Record runs

.gitignore

@@ -24,4 +24,6 @@ cov/*
 out
 .idea
 clojush.iml
-.gorilla-port
+.gorilla-port
+/clojush/
+.ipynb_checkpoints/

project.clj

@@ -1,6 +1,6 @@
 (defproject clojush "2.32.0-1-SNAPSHOT" 
   :description "The Push programming language and the PushGP genetic programming
-                system implemented in Clojure. See http://pushlanguage.com" 
+                system implemented in Clojure. See http://pushlanguage.com"
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :dependencies [[org.clojure/clojure "1.8.0"]
@@ -12,8 +12,9 @@
                  [clojure-csv "2.0.1"]
                  [org.clojure/data.json "0.2.6"]
                  [clj-random "0.1.7"]
-                 ;; https://mvnrepository.com/artifact/org.apache.commons/commons-math3 
-                 [org.apache.commons/commons-math3 "3.2"]]
+                 ;; https://mvnrepository.com/artifact/org.apache.commons/commons-math3
+                 [org.apache.commons/commons-math3 "3.2"]
+                 [cheshire "5.7.1"]]
   :plugins [[lein-codox "0.9.1"]
             [lein-shell "0.5.0"]
             [lein-gorilla "0.4.0"]
@@ -53,4 +54,3 @@
   ;;"-XX:+UseG1GC"
   ;:jvm-opts ["-Xmx12g" "-Xms12g" "-XX:+UseParallelGC"]
   :main clojush.core)
-

run-fly

@@ -25,7 +25,6 @@ homedir="/home/${fly_user}"
 rundir="$homedir/runs/$label-$number"
 repodir="$rundir/Clojush"
 outputdir="$rundir/output"
-
 ssh ${fly_user}@fly.hampshire.edu "mkdir -p $rundir"
 
 rsync  \
@@ -49,7 +48,7 @@ ssh ${fly_user}@fly.hampshire.edu /opt/pixar/tractor-blade-1.7.2/tractor-spool.p
   --jobcwd="${repodir}" \
   --priority=1 \
   --range 1-${n} \
-  -c "bash -c 'env JAVA_CMD=/usr/java/latest/bin/java /share/apps/bin/lein run $lein_command >  $outputdir/RANGE.out 2> $outputdir/RANGE.err'"
+  -c "bash -c 'env JAVA_CMD=/usr/java/latest/bin/java /share/apps/bin/lein run $lein_command :label $label >  $outputdir/RANGE.out 2> $outputdir/RANGE.err'"
 
 
 echo "Job ID: ${number}"

src/clojush/args.clj

@@ -411,7 +411,7 @@
           ;; When true, will exit the run when there is an individual with a zero-error vector
 
           ;;----------------------------------------
-          ;; Arguments related to printing JSON, EDN, or CSV logs
+          ;; Arguments related to printing JSON, EDN, CSV, and remote recording
           ;;----------------------------------------
 
           :print-csv-logs false
@@ -450,6 +450,14 @@
 
           :json-log-program-strings false
           ;; If true, JSON logs will include program strings for each individual.
+
+          :record-host nil
+          ;; Should be in the format "<hostname>:<port>"
+          ;; If set, will send logs of each run to a server running on this
+          ;; host
+          :label nil
+          ;; If set, will send this in the configuration of the run, to the
+          ;; external record
           )))
 
 (defn load-push-argmap

src/clojush/core.clj

@@ -16,6 +16,7 @@
 ;; for more details.
 
 (ns clojush.core
+  (:require [clojush.pushgp.record :as r])
   (:use [clojush.pushgp pushgp report])
   (:gen-class))
 
@@ -30,12 +31,13 @@
    This allows one to run an example with a call from the OS shell prompt like:
        lein run examples.simple-regression :population-size 3000"
   [& args]
+  (r/new-run!)
   (println "Command line args:" (apply str (interpose \space args)))
   (let [param-list (map #(if (.endsWith % ".ser")
                            (str %)
                            (read-string %))
                         (rest args))]
-    (require (symbol (first args)))
+    (require (symbol (r/config-data! [:problem-file] (first args))))
     (let [example-params (eval (symbol (str (first args) "/argmap")))
           params (merge example-params (apply sorted-map param-list))]
       (println "######################################")

src/clojush/problems/software/replace_space_with_newline.clj

@@ -160,6 +160,8 @@
     (println "Outputs of best individual on training cases:")
     (error-function best-program :train true)
     (println ";;******************************")
+    ;; return best individual with tests errors added so that those are recorded
+    (assoc best :test-errors best-test-errors)
     )) ;; To do validation, could have this function return an altered best individual
        ;; with total-error > 0 if it had error of zero on train but not on validation
        ;; set. Would need a third category of data cases, or a defined split of training cases.

src/clojush/pushgp/pushgp.clj

@@ -1,7 +1,8 @@
 (ns clojush.pushgp.pushgp
   (:require [clojure.java.io :as io]
             [clj-random.core :as random]
-            [clojure.repl :as repl])
+            [clojure.repl :as repl]
+            [clojush.pushgp.record :as r])
   (:use [clojush args globals util pushstate random individual evaluate simplification translate]
         [clojush.instructions boolean code common numbers random-instructions string char vectors
          tag zip return input-output genome]
@@ -144,13 +145,18 @@
   ([args]
     (reset! timer-atom (System/currentTimeMillis))
     (load-push-argmap args)
+    (when (some? (:record-host @push-argmap))
+      (r/host! (str (:record-host @push-argmap))))
     (random/with-rng (random/make-mersennetwister-rng (:random-seed @push-argmap))
       ;; set globals from parameters
       (reset-globals)
       (initial-report @push-argmap) ;; Print the inital report
-      (print-params @push-argmap)
+      (r/uuid! (:run-uuid @push-argmap))
+      (print-params (r/config-data! [:argmap] (dissoc @push-argmap :run-uuid)))
       (check-genetic-operator-probabilities-add-to-one @push-argmap)
       (timer @push-argmap :initialization)
+      (when (:print-timings @push-argmap)
+        (r/config-data! [:initialization-ms] (:initialization @timer-atom)))
       (println "\n;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")
       (println "\nGenerating initial population...") (flush)
       (let [pop-agents (make-pop-agents @push-argmap)
@@ -161,6 +167,7 @@
         ;(println)
         ;; Main loop
         (loop [generation 0]
+          (r/new-generation! generation)
           (println "Processing generation:" generation) (flush)
           (population-translate-plush-to-push pop-agents @push-argmap)
           (timer @push-argmap :reproduction)
@@ -183,6 +190,8 @@
           ;; report and check for success
           (let [[outcome best] (report-and-check-for-success (vec (doall (map deref pop-agents)))
                                                              generation @push-argmap)]
+            (r/generation-data! [:outcome] outcome)
+            (r/end-generation!)
             (cond (= outcome :failure) (do (printf "\nFAILURE\n")
                                          (if (:return-simplified-on-failure @push-argmap)
                                            (auto-simplify best 

src/clojush/pushgp/record.clj

@@ -0,0 +1,114 @@
+;;; Records the results of runs to an external server
+
+;; Use documented in https://push-language.hampshire.edu/t/recording-and-analyzing-experimental-results/830
+
+;; If `record-host` is set in the arguments, then we should send
+;; send data about each run, as it progresses, to that host for archival
+;; and monitoring purposes.
+
+;; The functions in this file are stateful and should be called in this order:
+;;
+;; (new-run! uuid! config-data!* (new-generation! generation-data!* end-generation!)*)*
+;;
+;; Currently it doesn't enforce this and if you call a method when you shouldn't
+;; the results are unkown.
+;; Also it will not send anything over the network until `host!` is called,
+;; before that, `end-generation!` will be a no-op.
+
+(ns clojush.pushgp.record
+  (:require [clojure.java.io]
+            [cheshire.core]
+            [cheshire.generate]
+            [clojure.string]))
+
+;; write functions as strings
+(cheshire.generate/add-encoder
+  clojure.lang.AFunction
+  cheshire.generate/encode-str)
+
+(def hostname-and-port (atom nil))
+(def writer (atom nil))
+
+(defn- ->writer
+  ; https://github.com/clojure-cookbook/clojure-cookbook/blob/master/05_network-io/5-09_tcp-client.asciidoc
+  []
+  (let [[hostname port] @hostname-and-port]
+    (-> (java.net.Socket. hostname port)
+      clojure.java.io/writer)))
+
+(defn- set-writer!
+  ; Tries to get a writer to send data on, and if it fails, retries every
+  ; 5 seconds
+  []
+  (println "Trying to connect to external server for recording at " @hostname-and-port "...")
+  (try
+    (reset! writer (->writer))
+    (catch java.net.ConnectException _
+      (Thread/sleep 5000)
+      (set-writer!))))
+
+(defn host! [host-str]
+  (let [[hostname port-str] (clojure.string/split host-str #":")]
+    (reset! hostname-and-port [hostname (int (bigint port-str))])
+    (set-writer!)))
+
+(defn- write-data! [data]
+  (when (some? @hostname-and-port)
+    (println "Trying to record data to external server...")
+    (try
+      (do
+        (cheshire.core/generate-stream data @writer)        
+        (.newLine @writer)
+        (.flush @writer))
+      (catch java.net.SocketException _
+        (set-writer!)
+        (write-data! data)))))
+
+(def data (atom {}))
+
+
+;; Stores a configuration option for the run, for the sequence of `ks` and value `v`
+;; i.e. (config-data! [:git-uuid] "abc-def")
+(defn config-data! [ks v]
+  (swap! data assoc-in (cons :config ks) v)
+  v)
+
+(defn seconds-since-epoch
+  ;; http://stackoverflow.com/a/17432411
+  ;; because Spark interprets numbers as dates in this format when in JSON
+  []
+  (quot (System/currentTimeMillis) 1000))
+
+;; Resets the run data and saves the start time. Should be called at the
+;; begining of a run
+(defn new-run! []
+  (reset! data {:config {:start-time (seconds-since-epoch)}}))
+
+(defn uuid! [uuid]
+  (swap! data assoc :uuid uuid))
+
+;; Resets the generation data and should be called at the begining of 
+;; each generation
+(defn new-generation! [index]
+  (swap!
+    data
+    assoc
+    :index index
+    :generation {:start-time (seconds-since-epoch)}))
+
+
+;; Stores data about the generation, i.e.
+;; (generation-data! [:best :error] [1 2 3 10])
+(defn generation-data! [ks v]
+  (swap! data assoc-in (cons :generation ks) v)
+  v)
+
+;; Sends the data for the current generation over the network to be recorded
+;; Also sends the configuration with each generation
+(defn end-generation! []
+  (let [{:keys [generation uuid index config]} @data]
+    (write-data!
+      (assoc generation
+        :config-uuid uuid
+        :index index
+        :config config))))