macro=; if ( argnumber < 1 ) clb ; quote ------------------------------------------------------------- This script contains instructions to run very large data sets. It builds trees and then starts improving them breaking into large pieces (it breaks them into a random number of pieces, between 8 and 14 by default). Every sector is analyzed (on the same machine, identified as myself), using xmult (and fusing with previous tree). As long as breaking it in pieces improves the tree in more than N steps, it continues improving it (up to 50 times). The instructions for breaking it are in the RUN_BIG_SECTORS routine (note that all the improved sec- tors are merged together, and globally swapped with TBR, which often improves the tree a little further). After finishing with the analysis for sectors (after two or more replications), then the improved tree is fused with the previous best tree(s). A new replication (outer loop) is then started. Up to 100 initial builds will be done. The trees produced are saved to file .built.tnt. The scores (with the time at which they were found) are saved to file scores. The files ver and peep contain further info which you can use to monitor a run. To run this in the background, type: tnt bground combosearch infile , & And you then monitor with tail -f peep ver scores NOTE: you must give the name of the data file as first arg. Arguments are: 1 - name of file with data ( mandatory ). 2 - minimum number of sectors in which to divide tree ( default is 8 ). 3 - maximum number of sectors in which to divide tree ( default is 14 ). 4 - 0/1 for beggining a new tree from scratch after completing the sectorial search [1], or dividing tree in some large sectors ignoring previous resolutions [0] (default is 0). 5 - minimum score gain in each cycle of division ( default is 5 ) --------------------------------------------------------------- ; view ; proc/; end ; p %1 ; goto = %0 ; ptnt commtime 1 ; ptnt hosts ; rs 0 ; log/; rs * ; drif: rfit 0.1 fitd 2 ; sec: xss 10-14+5-3 self 50 maxs 500 godrif 250 fuse 2 drift 8 ; rep +180/1/1 ; var = 10 best_global_score + prevscore + curscore + numfails + drop_nittree + minnumsecs + maxnumsecs + scratchit + minLgain ; if ( argnumber < 2 ) set minnumsecs 8 ; else set minnumsecs %2 ; end if ( argnumber < 3 ) set maxnumsecs 14 ; else set maxnumsecs %3 ; end if ( 'minnumsecs' > 'maxnumsecs' ) set 0 'minnumsecs' ; set minnumsecs 'maxnumsecs' ; set maxnumsecs '0' ; end if ( argnumber < 4 ) set scratchit 0 ; else set scratchit %4 ; end if ( argnumber < 5 ) set minLgain 5 ; else set minLgain %5 ; end system rm -f scores^ ; system rm -f .built.tnt^ ; system rm -f ver^ ; system rm -f peep^ ; log ver ; rseed ; macseed 0 ; set drop_nittree 0 ; set best_global_score 10000000000 ; macfloat 0 ; lquote [ ; loop 1 100 keep 1 ; set 0 time ; quote ------------------------------------------; quote NOW STARTING BUILD NR. #1 ('0' SECS.); quote ------------------------------------------; if ( ( #1 == 1 ) || 'scratchit' ) /* this starts all over */ xmu = rep 1 xss keep nofuse xmix prvmix ; else /* this re-starts tree in separate chunks */ set 0 getrandom[ 8 10 ] ; /* (its a little faster than from scratch, */ set drop_nittree 1 ; /* but retains some of the tree-structure) */ goto RUN_BIG_SECTORS '0' 2 0 ; set drop_nittree 0 ; end if ( ntrees > 0 ) tchoose 1 ; end log/; set numfails 0 ; loop 1 50 log/; log + ver ; set 0 time ; quote &10&10&10-------> STARTING IMPRO #2 (BUILD #1) -('0' SECS.)--------; set 0 getrandom[ 'minnumsecs' 'maxnumsecs' ] ; set prevscore length[0] ; goto RUN_BIG_SECTORS '0' 2 0 ; best ; set curscore length[0] ; set 0 ntrees ; tchoo '0' ; set 0 'prevscore' - 'curscore' ; set 1 time ; log/; log + ver ; quote &10&10------> FINISHED IMPRO #2 (saved '0' steps) - ('1' SECS.)... ; log/; if ( ( 'prevscore' - 'curscore' ) < 'minLgain' ) set numfails ++ ; if ( 'numfails' > 3 ) endloop ; end end stop log/; log + ver ; set 1 time ; quote AFTER IMPRO 'curscore', BEST SCORE: 'best_global_score' ('1' SECS.); tsave .built.tnt + ; save ; tsave/; if ( #1 > 1 ) keep 0 ; sho .built.tnt ; hold + 10 ; set 0 ntrees + 1 ; if ( '0' < 4 ) set 0 3 ; else set 0 5 ; end log/; log + ver ; quote BUILD NR. #1, about to fuse...; tfuse = round 4 nobestst ; unique ; best ; set 0 length[0] ; set 1 time ; set 2 0 ; if ( '0' < 'best_global_score' ) tsave .built.tnt +; save ; tsave/; set 2 'best_global_score' - '0' ; set best_global_score '0' ; end log/; log + scores ; quote BUILD #1 - AFTER FUSION: '0' ('1' SECS.) -- fusing saved '2' steps!!; log/; end stop quit ; /*****************************************************************/ RUN_BIG_SECTORS takes two arguments: 1) number of sectors 2) number of sectors to spawn together 3) reference tree/build [4) search level ] /*****************************************************************/ label RUN_BIG_SECTORS ; var = 10 num_sectors + num_joint_sectors + num_launches + launch_list[20] + launch_start + tree_to_improve + launch_started + search_level ; rseed * ; if ( argnumber > 3 ) set search_level %4 ; else set search_level 1.5 ; end bb: skipspr nosafe ; sect: xbuf ; if ( eqstring [ %3 build ] ) keep 0 ; mu1=ho1 ; sect= fuse 2 maxs 150 mins 50 global 200 godrift 300 self 40 ; set tree_to_improve 0 ; else if ( ntrees < 0 ) errmsg ---> No trees to improve!!: end ; set tree_to_improve %3 ; end quote NOW RUNNING WITH ARGS %1 %2 %3 ; set num_sectors %1 ; set num_joint_sectors %2 ; macfloat 0 ; set num_launches ( 'num_sectors' + 'num_joint_sectors' - 1 ) / 'num_joint_sectors'; set launch_start 0 ; copytree 'tree_to_improve' ; /* <--- make tree to improve the last one... */ coll notemp ; report- ; report + 60 ; if ( 'drop_nittree' ) log/; log + scores ; quote STARTING TREE ALL OVER (in %1 pieces, 'num_launches' launches)... ; log/; log + peep ; end loop 1 'num_launches' set 0 0 ; set launch_started 'launch_start' ; loop 1 'num_joint_sectors' if ( 'launch_start' == 'num_sectors' ) endloop ; end set launch_list [ '0' ] 'launch_start' ; set launch_start ++ ; set 0 ++ ; stop if ( !'0' ) quote LAUNCHED ALL JOBS! ; endloop ; end quote NOW LAUNCHING '0' JOBS: 'launch_list[0-('0'-1)]' ; ptnt begin SecSch 'num_sectors' myself /sector *'num_sectors' [ 'launch_list[0-('0'-1)]' ] 'tree_to_improve' /ram x1.33 = if ( 'drop_nittree' ) keep 0 ; end bb: skipspr nosafe ; hold 100 ; /* Here is the search; replace and do it as you like! */ macfloat 0 ; if ( ntax > 750 ) set 1 ntax / 200 ; set 2 ntax / 80 ; if ( "2" > 32 ) set 2 32 ; end sect : fuse 2 xss "1"-"2"+3 ; else sect: fuse 2 ; end tfuse: rounds 5 ; xmult = keep level 'search_level' css xss ; hold + 10 ; if ( !'drop_nittree' ) tfuse ; best ; end ; tchoose/; return polytree ; set 1 ntrees ; /* <--- in every cycle, combined tree is the last one... */ ptnt wait SecSch ; ptnt get !. ; loop 0 ( '0' - 1 ) silent = buffer file ; if ( havetgp[ TreeSector_#2_ ] == 0 ) continue ; end /* i.e. when reception failed, skip... */ hold + 5 ; freqdif * { TreeSector_#2_ } '1' ; tchoose .-'1' ; /* now dump it */ set 1 ntrees ; set 2 score['1'] ; if ( '2' < 'best_global_score' ) set best_global_score '2' ; end silent - buffer file ; set 3 time ; log/; log + scores ; quote ----> COMBINED PART 'launch_started' OF 'num_sectors', '2' STEPS, BEST SCORE 'best_global_score' ('3' SECS.); log/; log + peep ; set launch_started ++ ; stop stop silent - buffer file ; if ( !bground ) report= ; end set 0 ntrees ; tchoose '0' ; bb = nomulp ; set 0 score[0] ; set 1 time ; if ( '0' < 'best_global_score' ) set best_global_score '0' ; end log/; log + scores ; if ( 'drop_nittree' ) quote ALL PARTS SCRAMBLED: '0' , BEST SCORE 'best_global_score' ('1' SECS.) ; else quote ALL PARTS FUSED: '0' , BEST SCORE 'best_global_score' ('1' SECS.) ; end log/; log + peep ; keep 1 ; proc/ ;