SCons integrationΒΆ

This SConstruct file is an example of using nestly with the SCons build system:

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
# -*- python -*-
#
# This example takes every file in the inputs directory and performs the
# following operations:
# * cuts out a column range from every line in the file; either 1-5 or 3-40
# * optionally filters out every line that has an "o" or "O"
# * runs wc on every such file
# * aggregate these together using the prep_tab.sh script
#
# Assuming that SCons is installed, you should be able to run this example by
# typing `scons` in this directory. That should build a series of things in the
# `build` directory. Because this is a build system, deleting a file or directory
# in the build directory and then running scons will simply rerun the needed parts.

from os.path import join
import os

from nestly.scons import SConsWrap
from nestly import Nest

env = Environment()

# Passing an argument to `alias_environment` allows building targets based on nest
# key.
# For example, the `counts` files described below can be built by invoking
# `scons counts`
nest = SConsWrap(Nest(), 'build', alias_environment=env)


# Add our aggregate targets, initializing collections that will get populated
# downstream. At the end of the pipeline, we will operate on these collections.
# The `add_argument` takes a key which will be the key used for accessing the
# collection. The `list` argument specifies that the collection will be a list.
nest.add_aggregate('count_agg', list)
nest.add_aggregate('cut_agg', list)

# Add a nest level with the name 'input_file' that takes the files in the inputs
# directory as its nestable list. Make its label function just the basename.
nest.add('input_file', [join('inputs', f) for f in os.listdir('inputs')],
      label_func=os.path.basename)

# This nest level determines the column range we will cut out of the file.
nest.add('cut_range', ['1-5', '3-40'])

# This adds a nest item with the name 'cut' and makes an SCons target out of
# the result.
@nest.add_target()
def cut(outdir, c):
    cut, = Command(join(outdir, 'cut'),
                   c['input_file'],
                   'cut -c {0[cut_range]} <$SOURCE >$TARGET'.format(c))
    # Here we add this cut file to the all_cut aggregator before returning
    c['cut_agg'].append(cut)
    return cut

# This nest level determines whether we remove the lines with o's.
nest.add('o_choice', ['remove_o', 'leave_o'])

@nest.add_target()
def o_choice(outdir, c):
    # If we leave the o lines, then we don't have to do anything.
    if c['o_choice'] == 'leave_o':
        return c['cut']
    # If we want to remove the o lines, then we have to make an SCons Command
    # that does so with sed.
    return Command(join(outdir, 'o_removed'),
                   c['cut'],
                   'sed "/[oO]/d" <$SOURCE >$TARGET')[0]

# Add a target for the word counts.
@nest.add_target()
def counts(outdir, c):
    counts, = Command(join(outdir, 'counts'),
                      c['o_choice'],
                      'wc <$SOURCE >$TARGET')
    # Add the resulting file to the count_agg collection
    c['count_agg'].append(counts)
    return counts

# Add a control dictionary with chosen values to each leaf directory
nest.add_controls(env)

# Before operating on our aggregate collections, we return back to the original
# nest level in which the aggregates were created by using the `pop` function to
# remove all of the later nest levels from the nest state, leaving only the
# collections.
nest.pop('input_file')

# Now, back at the initial nest level, we can operate on the populated aggregate
# collections. First, the counts:
@nest.add_target()
def all_counts(outdir, c):
    return Command(join(outdir, 'all_counts.tab'),
                   c['count_agg'],
                   './prep_tab.sh $SOURCES | column -t >$TARGET')

# Then the cuts:
@nest.add_target()
def all_cut(outdir, c):
    return Command(join(outdir, 'all_cut.txt'),
                   c['cut_agg'],
                   'cat $SOURCES >$TARGET')