Record the machine that produced each library's results

resultsdb.py

@@ -229,14 +229,19 @@ def createTables(self, branch):
     if user_version == 0:
       # Table to lookup from a run (date, branch) to omcversion used
       cursor.execute("CREATE TABLE if not exists [omcversion] (date integer NOT NULL, branch text NOT NULL, omcversion text NOT NULL)")
-      # Table to lookup from a run (date, branch) which library versions were used
-      cursor.execute("CREATE TABLE if not exists [libversion] (date integer NOT NULL, branch text NOT NULL, libname text NOT NULL, libversion text NOT NULL, confighash integer NOT NULL)")
+      # Table to lookup from a run (date, branch) which library versions were used,
+      # and the machine that produced them
+      cursor.execute("CREATE TABLE if not exists [libversion] (date integer NOT NULL, branch text NOT NULL, libname text NOT NULL, libversion text NOT NULL, confighash integer NOT NULL, host text, sysinfo text)")
     elif user_version == 1:
       cursor.execute("ALTER TABLE [libversion] ADD COLUMN confighash integer NOT NULL DEFAULT(0)")
+      self.addLibversionHost(cursor)
     elif user_version == 2:
       for tbl in [t for t in self.tables() if t not in ["libversion", "omcversion"]]:
         cursor.execute("ALTER TABLE [%s] ADD COLUMN parsing real NOT NULL DEFAULT(0.0)" % tbl)
-    elif user_version != 3:
+      self.addLibversionHost(cursor)
+    elif user_version == 3:
+      self.addLibversionHost(cursor)
+    elif user_version != 4:
       raise SystemExit("Unknown schema user_version=%d" % user_version)
 
     cols = ", ".join("%s %s NOT NULL" % (c, SQLITE_TYPES[t]) for c, t in BRANCH_COLUMNS)
@@ -245,7 +250,18 @@ def createTables(self, branch):
     cursor.execute("DROP INDEX IF EXISTS [idx_%s_date]" % branch)
     cursor.execute("DROP INDEX IF EXISTS idx_omcversion_date")
     cursor.execute("DROP INDEX IF EXISTS idx_libversion_date")
-    self.setUserVersion(3)
+    self.setUserVersion(4)
+
+  def addLibversionHost(self, cursor):
+    """Add the host columns to an existing [libversion].
+
+    ADD COLUMN only, so the rows already there keep their results and simply
+    read back NULL for a run whose machine was never recorded.
+    """
+    have = set(r[1] for r in cursor.execute("PRAGMA table_info([libversion])"))
+    for col in ["host", "sysinfo"]:
+      if col not in have:
+        cursor.execute("ALTER TABLE [libversion] ADD COLUMN %s text" % col)
 
   def tables(self):
     return [t for (t,) in self.conn.execute("SELECT name FROM sqlite_master WHERE type='table'")]
@@ -399,7 +415,12 @@ def createTables(self, branch):
         date bigint NOT NULL, branch text NOT NULL, omcversion text)""")
     cursor.execute("""CREATE TABLE IF NOT EXISTS libversion (
         date bigint NOT NULL, branch text NOT NULL, libname text NOT NULL,
-        libversion text, confighash bigint NOT NULL)""")
+        libversion text, confighash bigint NOT NULL,
+        host text, sysinfo text)""")
+    # The shared database predates the host columns; add them without touching
+    # the rows already in there, which keep their results and read back NULL.
+    for col in ["host", "sysinfo"]:
+      cursor.execute("ALTER TABLE libversion ADD COLUMN IF NOT EXISTS %s text" % col)
     cols = ", ".join("%s %s%s" % (c, POSTGRES_TYPES[t], " NOT NULL" if c in BRANCH_KEY else "")
                      for c, t in BRANCH_COLUMNS)
     cursor.execute("CREATE TABLE IF NOT EXISTS %s (%s)" % (self.quote(branch), cols))

sqlite2postgres.py

@@ -69,7 +69,8 @@
 LOOKUP_COLUMNS = {
     "omcversion": [("date", "bigint"), ("branch", "text"), ("omcversion", "text")],
     "libversion": [("date", "bigint"), ("branch", "text"), ("libname", "text"),
-                   ("libversion", "text"), ("confighash", "bigint")],
+                   ("libversion", "text"), ("confighash", "bigint"),
+                   ("host", "text"), ("sysinfo", "text")],
 }
 
 # Derived data, no longer generated by all-plots.py; not worth migrating.
@@ -233,8 +234,12 @@ def migrate_table(pg, sconn, source, tbl, columns, batch, quiet, skip_existing=F
   names = [c for c, _ in columns]
   have = sqlite_columns(sconn, tbl)
   missing = [c for c in names if c not in have]
-  # Older databases lack "parsing"; test.py defaults it to 0.0 as well.
-  select = ",".join(ident(c) if c in have else "0" for c in names)
+  # Older databases lack "parsing", and ones older still lack the libversion
+  # host columns.  test.py defaults the numbers to 0; a text column a run never
+  # recorded is NULL, not the string "0".
+  types = dict(columns)
+  select = ",".join(ident(c) if c in have else ("NULL" if types[c] == "text" else "0")
+                    for c in names)
 
   create_table(pg, tbl, columns)
   last_rowid, rows_read, done = read_progress(pg, source, tbl)

test.py

@@ -12,6 +12,7 @@
 from joblib import Parallel, delayed
 import simplejson as json
 import psutil, subprocess, threading, hashlib
+import socket
 from subprocess import call
 from monotonic import monotonic
 from omcommon import friendlyStr, multiple_replace
@@ -993,6 +994,26 @@ def resultValues(model, libname, data, simulator=None):
     data.get("parsing") or 0.0
   )
 
+def cpu_name():
+  if isWin:
+    return processor()
+  else:
+    for line in open("/proc/cpuinfo").readlines():
+      if "model name" in line.strip():
+        return (re.sub( ".*model name.*:", "", line, count=1)).strip()
+
+if isWin:
+  lsb_release = ""
+else:
+  try:
+    lsb_release = check_output_log(commands + ["cat","/etc/lsb-release"]).decode().strip()
+    lsb_release = dict(a.split("=") for a in lsb_release.split("\n"))["DISTRIB_DESCRIPTION"].strip('"')
+  except:
+    lsb_release = ""
+
+hostname = socket.gethostname()
+sysInfo = "%s: %s, %d GB RAM, %s%s" % (hostname, cpu_name(), int(math.ceil(psutil.virtual_memory().total / (1024.0**3))), ("Docker " + docker + " ") if docker else "", lsb_release)
+
 for (resultBranch, simulator) in resultBranches:
   db.createTables(resultBranch)
   for key in stats.keys():
@@ -1003,7 +1024,7 @@ def resultValues(model, libname, data, simulator=None):
                    resultValues(model, libname, data, simulator))
   for libname in stats_by_libname.keys():
     confighash = stats_by_libname[libname]["conf"]["confighash"]
-    cursor.execute("INSERT INTO libversion VALUES (?,?,?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash))
+    cursor.execute("INSERT INTO libversion VALUES (?,?,?,?,?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash, hostname, sysInfo))
   cursor.execute("INSERT INTO omcversion VALUES (?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, omc_version))
 """
 # Not really a good thing to do; was just done to make generation of the report simpler
@@ -1028,25 +1049,6 @@ def checkPhase(phase, n):
 def is_non_zero_file(fpath):
     return os.path.isfile(os.path.normpath(fpath)) and os.path.getsize(os.path.normpath(fpath)) > 0
 
-def cpu_name():
-  if isWin:
-    return processor()
-  else:
-    for line in open("/proc/cpuinfo").readlines():
-      if "model name" in line.strip():
-        return (re.sub( ".*model name.*:", "", line, count=1)).strip()
-
-if isWin:
-  lsb_release = ""
-else:
-  try:
-    lsb_release = check_output_log(commands + ["cat","/etc/lsb-release"]).decode().strip()
-    lsb_release = dict(a.split("=") for a in lsb_release.split("\n"))["DISTRIB_DESCRIPTION"].strip('"')
-  except:
-    lsb_release = ""
-
-sysInfo = "%s, %d GB RAM, %s%s" % (cpu_name(), int(math.ceil(psutil.virtual_memory().total / (1024.0**3))), ("Docker " + docker + " ") if docker else "", lsb_release)
-
 # create target dir to move results without sync operations (win or when --noSync is used)
 def stageRootFor(simulator, suffix):
   """The directory a branch is published from.