ClusterShell.CLI.Nodeset

1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2008-2016 CEA/DAM 4 # Copyright (C) 2015-2016 Stephane Thiell <sthiell@stanford.edu> 5 # 6 # This file is part of ClusterShell. 7 # 8 # ClusterShell is free software; you can redistribute it and/or 9 # modify it under the terms of the GNU Lesser General Public 10 # License as published by the Free Software Foundation; either 11 # version 2.1 of the License, or (at your option) any later version. 12 # 13 # ClusterShell is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 # Lesser General Public License for more details. 17 # 18 # You should have received a copy of the GNU Lesser General Public 19 # License along with ClusterShell; if not, write to the Free Software 20 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 21 22 """ 23 compute advanced nodeset operations 24 25 The nodeset command is an utility command provided with the 26 ClusterShell library which implements some features of the NodeSet 27 and RangeSet classes. 28 """ 29 30 import logging 31 import math 32 import sys 33 import random 34 35 from ClusterShell.CLI.Error import GENERIC_ERRORS, handle_generic_error 36 from ClusterShell.CLI.OptionParser import OptionParser 37 from ClusterShell.CLI.Utils import NodeSet # safe import 38 39 from ClusterShell.NodeSet import RangeSet, grouplist, std_group_resolver 40 from ClusterShell.NodeUtils import GroupSourceNoUpcall 41 42

43 -def process_stdin(xsetop, xsetcls, autostep):

44 """Process standard input and operate on xset.""" 45 # Build temporary set (stdin accumulator) 46 tmpset = xsetcls(autostep=autostep) 47 for line in sys.stdin.readlines(): 48 # Support multi-lines and multi-nodesets per line 49 line = line[0:line.find('#')].strip() 50 for elem in line.split(): 51 # Do explicit object creation for RangeSet 52 tmpset.update(xsetcls(elem, autostep=autostep)) 53 # Perform operation on xset 54 if tmpset: 55 xsetop(tmpset)

56

57 -def compute_nodeset(xset, args, autostep):

58 """Apply operations and operands from args on xset, an initial 59 RangeSet or NodeSet.""" 60 class_set = xset.__class__ 61 # Process operations 62 while args: 63 arg = args.pop(0) 64 if arg in ("-i", "--intersection"): 65 val = args.pop(0) 66 if val == '-': 67 process_stdin(xset.intersection_update, class_set, autostep) 68 else: 69 xset.intersection_update(class_set(val, autostep=autostep)) 70 elif arg in ("-x", "--exclude"): 71 val = args.pop(0) 72 if val == '-': 73 process_stdin(xset.difference_update, class_set, autostep) 74 else: 75 xset.difference_update(class_set(val, autostep=autostep)) 76 elif arg in ("-X", "--xor"): 77 val = args.pop(0) 78 if val == '-': 79 process_stdin(xset.symmetric_difference_update, class_set, 80 autostep) 81 else: 82 xset.symmetric_difference_update(class_set(val, 83 autostep=autostep)) 84 elif arg == '-': 85 process_stdin(xset.update, xset.__class__, autostep) 86 else: 87 xset.update(class_set(arg, autostep=autostep)) 88 89 return xset

90

91 -def print_source_groups(source, level, xset, opts):

92 """ 93 Print groups from a source, a level of verbosity and an optional 94 nodeset acting as a filter. 95 """ 96 # list groups of some specified nodes? 97 if opts.all or xset or opts.and_nodes or opts.sub_nodes or opts.xor_nodes: 98 # When some node sets are provided as argument, the list command 99 # retrieves node groups these nodes belong to, thanks to the 100 # groups() method. 101 # Note: stdin support is enabled when '-' is found. 102 groups = xset.groups(source, opts.groupbase) 103 for group, (gnodes, inodes) in groups.iteritems(): 104 if level == 1: 105 print group 106 elif level == 2: 107 print "%s %s" % (group, inodes) 108 else: 109 print "%s %s %d/%d" % (group, inodes, len(inodes), 110 len(gnodes)) 111 else: 112 # "raw" group list when no argument at all 113 for group in grouplist(source): 114 if source and not opts.groupbase: 115 nsgroup = "@%s:%s" % (source, group) 116 else: 117 nsgroup = "@%s" % group 118 if level == 1: 119 print nsgroup 120 else: 121 nodes = NodeSet(nsgroup) 122 if level == 2: 123 print "%s %s" % (nsgroup, nodes) 124 else: 125 print "%s %s %d" % (nsgroup, nodes, len(nodes))

126

127 -def command_list(options, xset, group_resolver):

128 """List command handler (-l/-ll/-lll/-L/-LL/-LLL).""" 129 list_level = options.list + options.listall 130 if options.listall: 131 # useful: sources[0] is always the default or selected source 132 sources = group_resolver.sources() 133 # do not print name of default group source unless -s specified 134 if not options.groupsource: 135 sources[0] = None 136 else: 137 sources = [options.groupsource] 138 139 for source in sources: 140 try: 141 print_source_groups(source, list_level, xset, options) 142 except GroupSourceNoUpcall, exc: 143 if not options.listall: 144 raise 145 # missing list upcall is not fatal with -L 146 msgfmt = "Warning: No %s upcall defined for group source %s" 147 print >>sys.stderr, msgfmt % (exc, source)

148

149 -def nodeset():

150 """script subroutine""" 151 class_set = NodeSet 152 usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]" 153 154 parser = OptionParser(usage) 155 parser.install_nodeset_commands() 156 parser.install_nodeset_operations() 157 parser.install_nodeset_options() 158 (options, args) = parser.parse_args() 159 160 group_resolver = std_group_resolver() 161 162 if options.debug: 163 logging.basicConfig(level=logging.DEBUG) 164 165 # Check for command presence 166 cmdcount = int(options.count) + int(options.expand) + \ 167 int(options.fold) + int(bool(options.list)) + \ 168 int(bool(options.listall)) + int(options.regroup) + \ 169 int(options.groupsources) 170 if not cmdcount: 171 parser.error("No command specified.") 172 elif cmdcount > 1: 173 parser.error("Multiple commands not allowed.") 174 175 if options.rangeset: 176 class_set = RangeSet 177 178 if options.all or options.regroup: 179 if class_set != NodeSet: 180 parser.error("-a/-r only supported in NodeSet mode") 181 182 if options.maxsplit is not None and options.contiguous: 183 parser.error("incompatible splitting options (split, contiguous)") 184 185 if options.maxsplit is None: 186 options.maxsplit = 1 187 188 if options.axis and (not options.fold or options.rangeset): 189 parser.error("--axis option is only supported when folding nodeset") 190 191 if options.groupsource and not options.quiet and class_set == RangeSet: 192 print >> sys.stderr, "WARNING: option group source \"%s\" ignored" \ 193 % options.groupsource 194 195 # We want -s <groupsource> to act as a substition of default groupsource 196 # (ie. it's not necessary to prefix group names by this group source). 197 if options.groupsource: 198 group_resolver.default_source_name = options.groupsource 199 200 # The groupsources command simply lists group sources. 201 if options.groupsources: 202 if options.quiet: 203 dispdefault = "" # don't show (default) if quiet is set 204 else: 205 dispdefault = " (default)" 206 for src in group_resolver.sources(): 207 print "%s%s" % (src, dispdefault) 208 dispdefault = "" 209 return 210 211 autostep = options.autostep 212 213 # Do not use autostep for computation when a percentage or the special 214 # value 'auto' is specified. Real autostep value is set post-process. 215 if type(autostep) is float or autostep == 'auto': 216 autostep = None 217 218 # Instantiate RangeSet or NodeSet object 219 xset = class_set(autostep=autostep) 220 221 if options.all: 222 # Include all nodes from external node groups support. 223 xset.update(NodeSet.fromall()) # uses default_source when set 224 225 if not args and not options.all and not (options.list or options.listall): 226 # No need to specify '-' to read stdin in these cases 227 process_stdin(xset.update, xset.__class__, autostep) 228 229 # Apply first operations (before first non-option) 230 for nodes in options.and_nodes: 231 if nodes == '-': 232 process_stdin(xset.intersection_update, xset.__class__, autostep) 233 else: 234 xset.intersection_update(class_set(nodes, autostep=autostep)) 235 for nodes in options.sub_nodes: 236 if nodes == '-': 237 process_stdin(xset.difference_update, xset.__class__, autostep) 238 else: 239 xset.difference_update(class_set(nodes, autostep=autostep)) 240 for nodes in options.xor_nodes: 241 if nodes == '-': 242 process_stdin(xset.symmetric_difference_update, xset.__class__, 243 autostep) 244 else: 245 xset.symmetric_difference_update(class_set(nodes, 246 autostep=autostep)) 247 248 # Finish xset computing from args 249 compute_nodeset(xset, args, autostep) 250 251 # The list command has a special handling 252 if options.list > 0 or options.listall > 0: 253 return command_list(options, xset, group_resolver) 254 255 # Interprete special characters (may raise SyntaxError) 256 separator = eval('\'%s\'' % options.separator, {"__builtins__":None}, {}) 257 258 if options.slice_rangeset: 259 _xset = class_set() 260 for sli in RangeSet(options.slice_rangeset).slices(): 261 _xset.update(xset[sli]) 262 xset = _xset 263 264 if options.autostep == 'auto': 265 # Simple implementation of --autostep=auto 266 # if we have at least 3 nodes, all index should be foldable as a-b/n 267 xset.autostep = max(3, len(xset)) 268 elif type(options.autostep) is float: 269 # at least % of nodes should be foldable as a-b/n 270 autofactor = float(options.autostep) 271 xset.autostep = int(math.ceil(float(len(xset)) * autofactor)) 272 273 # user-specified nD-nodeset fold axis 274 if options.axis: 275 if not options.axis.startswith('-'): 276 # axis are 1-indexed in nodeset CLI (0 ignored) 277 xset.fold_axis = tuple(x-1 for x in RangeSet(options.axis) if x > 0) 278 else: 279 # negative axis index (only single number supported) 280 xset.fold_axis = [int(options.axis)] 281 282 if options.pick and options.pick < len(xset): 283 # convert to string for sample as nsiter() is slower for big 284 # nodesets; and we assume options.pick will remain small-ish 285 keep = random.sample(xset, options.pick) 286 # explicit class_set creation and str() convertion for RangeSet 287 keep = class_set(','.join([str(x) for x in keep])) 288 xset.intersection_update(keep) 289 290 fmt = options.output_format # default to '%s' 291 292 # Display result according to command choice 293 if options.expand: 294 xsubres = lambda x: separator.join((fmt % s for s in x.striter())) 295 elif options.fold: 296 # Special case when folding using NodeSet and format is set (#277) 297 if class_set is NodeSet and fmt != '%s': 298 # Create a new set after format has been applied to each node 299 xset = class_set._fromlist1((fmt % xnodestr for xnodestr in xset), 300 autostep=xset.autostep) 301 xsubres = lambda x: x 302 else: 303 xsubres = lambda x: fmt % x 304 elif options.regroup: 305 xsubres = lambda x: fmt % x.regroup(options.groupsource, 306 noprefix=options.groupbase) 307 else: 308 xsubres = lambda x: fmt % len(x) 309 310 if not xset or options.maxsplit <= 1 and not options.contiguous: 311 print xsubres(xset) 312 else: 313 if options.contiguous: 314 xiterator = xset.contiguous() 315 else: 316 xiterator = xset.split(options.maxsplit) 317 for xsubset in xiterator: 318 print xsubres(xsubset)

319

320 -def main():

321 """main script function""" 322 try: 323 nodeset() 324 except (AssertionError, IndexError, ValueError), ex: 325 print >> sys.stderr, "ERROR:", ex 326 sys.exit(1) 327 except SyntaxError: 328 print >> sys.stderr, "ERROR: invalid separator" 329 sys.exit(1) 330 except GENERIC_ERRORS, ex: 331 sys.exit(handle_generic_error(ex)) 332 333 sys.exit(0)

334 335 336 if __name__ == '__main__': 337 main() 338

Source Code for Module ClusterShell.CLI.Nodeset