|
| 1 | +(ns clj-frontend.demo |
| 2 | + (:use drake.clj-frontend)) |
| 3 | + |
| 4 | +;; minimal workflow example |
| 5 | + |
| 6 | +;; Define a workflow called minimal-workflow. |
| 7 | +(def minimal-workflow |
| 8 | + (-> |
| 9 | + (new-workflow) ;Create a new workflow |
| 10 | + (cmd-step ;Add a command step with the |
| 11 | + ;following arguments |
| 12 | + ["out"] ;Array of outputs |
| 13 | + [] ;Array of inputs |
| 14 | + ["echo \"We are writing to a file here\" > $OUTPUT"] ;Array of commands |
| 15 | + ))) |
| 16 | + |
| 17 | +;; To see what will happen without running the workflow uncomment the |
| 18 | +;; following line or run it at in the repl: |
| 19 | + |
| 20 | +;; (run-workflow minimal-workflow :preview true) |
| 21 | + |
| 22 | +;; Use the following line to actually run the workflow |
| 23 | + |
| 24 | +;; (run-workflow minimal-workflow) |
| 25 | + |
| 26 | + |
| 27 | +;; A more advanced example |
| 28 | + |
| 29 | +(def advanced-workflow |
| 30 | + (-> |
| 31 | + (new-workflow) |
| 32 | + (cmd-step |
| 33 | + ["out1" |
| 34 | + "out2"] |
| 35 | + [] |
| 36 | + ["echo \"This is the first output.\" > $OUTPUT0" |
| 37 | + "echo \"This is the second output.\" > $OUTPUT1"] ;multiple commands |
| 38 | + :timecheck false) ;options are key value pairs |
| 39 | + (method |
| 40 | + "test_method" |
| 41 | + ["echo \"Here we are using a method.\" > $OUTPUT"]) |
| 42 | + (method-step |
| 43 | + ["out_method"] ;outputs |
| 44 | + [] ;inputs |
| 45 | + "test_method") ;method name |
| 46 | + (set-var "test_var" "TEST_VAR_VALUE") ;var name, var value |
| 47 | + (set-var "output_three" "out3") |
| 48 | + (cmd-step |
| 49 | + ["$[output_three]"] ;inputs and outputs can have |
| 50 | + ;$[XXX] substitution |
| 51 | + ["out1" "%a_tag"] ;tags are allowed in inputs |
| 52 | + ;and outputs |
| 53 | + ;; $[XXX] substitution is allowed in commands. |
| 54 | + ["echo \"This is the third output.\" > $OUTPUT" |
| 55 | + "echo \"test_var is set to $test_var - $[test_var].\" >> $OUTPUT" |
| 56 | + "echo \"The file $INPUT contains:\" | cat - $INPUT >> $[OUTPUT]"]))) |
| 57 | + |
| 58 | +;; (run-workflow advanced-workflow :preview true) |
| 59 | +;; (run-workflow advanced-workflow) |
| 60 | + |
| 61 | +;; Example with reduce |
| 62 | + |
| 63 | +;; Let's say you want to take several raw data sources from the |
| 64 | +;; internet and for each source you want to create a directory, |
| 65 | +;; download some data into it, and do several processing steps on the |
| 66 | +;; data. We will express this as a map called dir->url-map between the |
| 67 | +;; directory names we want to create and the raw data sources we want |
| 68 | +;; to process. |
| 69 | + |
| 70 | +(def dir->url-map |
| 71 | + "Hash map of: |
| 72 | + Directory Names => URLs" |
| 73 | + {"Dir1" "http://url1" |
| 74 | + "Dir2" "http://url2" |
| 75 | + "Dir3" "http://url3"}) |
| 76 | + |
| 77 | +;; Now we need a function that takes an existing workflow and adds new |
| 78 | +;; steps to it for each directory => url pair from our data-map. |
| 79 | + |
| 80 | + |
| 81 | +(defn download-and-process |
| 82 | + "I take an existing workflow and download and process the data at |
| 83 | + url into the directory dir" |
| 84 | + [w-flow [dir url]] ;note the argument |
| 85 | + ;destructuring |
| 86 | + (-> w-flow |
| 87 | + (base "") ;make sure we are in top |
| 88 | + ;directory |
| 89 | + (cmd-step |
| 90 | + [dir] |
| 91 | + [] |
| 92 | + ["mkdir -p $OUTPUT"]) |
| 93 | + (base dir) ;move into dir for our |
| 94 | + ;subsequent commands |
| 95 | + (cmd-step |
| 96 | + ["raw_data"] |
| 97 | + [] |
| 98 | + ["wget -O $OUTPUT " url] ;get the data |
| 99 | + :timecheck false) |
| 100 | + (cmd-step |
| 101 | + ["sorted_data"] |
| 102 | + ["raw_data"] |
| 103 | + ["sort -o $OUTPUT"]) ;sort the data |
| 104 | + ;; more steps can be added here |
| 105 | + )) |
| 106 | + |
| 107 | +;; Finally we can use `reduce` with `download-and-process` to add |
| 108 | +;; several workflow steps for each dir => url pair in dir->url-map. |
| 109 | + |
| 110 | +(def reduce-workflow |
| 111 | + (reduce |
| 112 | + download-and-process |
| 113 | + (new-workflow) |
| 114 | + dir->url-map)) |
| 115 | + |
| 116 | +;; (run-workflow reduce-workflow :preview true) |
| 117 | + |
| 118 | +;; this is a fake workflow in that the interet data doesn't exist so |
| 119 | +;; we can't acutally run it |
0 commit comments