diff --git a/resultsdb.py b/resultsdb.py index 81bc764..e709038 100644 --- a/resultsdb.py +++ b/resultsdb.py @@ -229,14 +229,19 @@ def createTables(self, branch): if user_version == 0: # Table to lookup from a run (date, branch) to omcversion used cursor.execute("CREATE TABLE if not exists [omcversion] (date integer NOT NULL, branch text NOT NULL, omcversion text NOT NULL)") - # Table to lookup from a run (date, branch) which library versions were used - cursor.execute("CREATE TABLE if not exists [libversion] (date integer NOT NULL, branch text NOT NULL, libname text NOT NULL, libversion text NOT NULL, confighash integer NOT NULL)") + # Table to lookup from a run (date, branch) which library versions were used, + # and the machine that produced them + cursor.execute("CREATE TABLE if not exists [libversion] (date integer NOT NULL, branch text NOT NULL, libname text NOT NULL, libversion text NOT NULL, confighash integer NOT NULL, host text, sysinfo text)") elif user_version == 1: cursor.execute("ALTER TABLE [libversion] ADD COLUMN confighash integer NOT NULL DEFAULT(0)") + self.addLibversionHost(cursor) elif user_version == 2: for tbl in [t for t in self.tables() if t not in ["libversion", "omcversion"]]: cursor.execute("ALTER TABLE [%s] ADD COLUMN parsing real NOT NULL DEFAULT(0.0)" % tbl) - elif user_version != 3: + self.addLibversionHost(cursor) + elif user_version == 3: + self.addLibversionHost(cursor) + elif user_version != 4: raise SystemExit("Unknown schema user_version=%d" % user_version) cols = ", ".join("%s %s NOT NULL" % (c, SQLITE_TYPES[t]) for c, t in BRANCH_COLUMNS) @@ -245,7 +250,18 @@ def createTables(self, branch): cursor.execute("DROP INDEX IF EXISTS [idx_%s_date]" % branch) cursor.execute("DROP INDEX IF EXISTS idx_omcversion_date") cursor.execute("DROP INDEX IF EXISTS idx_libversion_date") - self.setUserVersion(3) + self.setUserVersion(4) + + def addLibversionHost(self, cursor): + """Add the host columns to an existing [libversion]. + + ADD COLUMN only, so the rows already there keep their results and simply + read back NULL for a run whose machine was never recorded. + """ + have = set(r[1] for r in cursor.execute("PRAGMA table_info([libversion])")) + for col in ["host", "sysinfo"]: + if col not in have: + cursor.execute("ALTER TABLE [libversion] ADD COLUMN %s text" % col) def tables(self): return [t for (t,) in self.conn.execute("SELECT name FROM sqlite_master WHERE type='table'")] @@ -399,7 +415,12 @@ def createTables(self, branch): date bigint NOT NULL, branch text NOT NULL, omcversion text)""") cursor.execute("""CREATE TABLE IF NOT EXISTS libversion ( date bigint NOT NULL, branch text NOT NULL, libname text NOT NULL, - libversion text, confighash bigint NOT NULL)""") + libversion text, confighash bigint NOT NULL, + host text, sysinfo text)""") + # The shared database predates the host columns; add them without touching + # the rows already in there, which keep their results and read back NULL. + for col in ["host", "sysinfo"]: + cursor.execute("ALTER TABLE libversion ADD COLUMN IF NOT EXISTS %s text" % col) cols = ", ".join("%s %s%s" % (c, POSTGRES_TYPES[t], " NOT NULL" if c in BRANCH_KEY else "") for c, t in BRANCH_COLUMNS) cursor.execute("CREATE TABLE IF NOT EXISTS %s (%s)" % (self.quote(branch), cols)) diff --git a/sqlite2postgres.py b/sqlite2postgres.py index 850d445..af49bb9 100644 --- a/sqlite2postgres.py +++ b/sqlite2postgres.py @@ -69,7 +69,8 @@ LOOKUP_COLUMNS = { "omcversion": [("date", "bigint"), ("branch", "text"), ("omcversion", "text")], "libversion": [("date", "bigint"), ("branch", "text"), ("libname", "text"), - ("libversion", "text"), ("confighash", "bigint")], + ("libversion", "text"), ("confighash", "bigint"), + ("host", "text"), ("sysinfo", "text")], } # Derived data, no longer generated by all-plots.py; not worth migrating. @@ -233,8 +234,12 @@ def migrate_table(pg, sconn, source, tbl, columns, batch, quiet, skip_existing=F names = [c for c, _ in columns] have = sqlite_columns(sconn, tbl) missing = [c for c in names if c not in have] - # Older databases lack "parsing"; test.py defaults it to 0.0 as well. - select = ",".join(ident(c) if c in have else "0" for c in names) + # Older databases lack "parsing", and ones older still lack the libversion + # host columns. test.py defaults the numbers to 0; a text column a run never + # recorded is NULL, not the string "0". + types = dict(columns) + select = ",".join(ident(c) if c in have else ("NULL" if types[c] == "text" else "0") + for c in names) create_table(pg, tbl, columns) last_rowid, rows_read, done = read_progress(pg, source, tbl) diff --git a/test.py b/test.py index 4b61852..8351f66 100755 --- a/test.py +++ b/test.py @@ -12,6 +12,7 @@ from joblib import Parallel, delayed import simplejson as json import psutil, subprocess, threading, hashlib +import socket from subprocess import call from monotonic import monotonic from omcommon import friendlyStr, multiple_replace @@ -993,6 +994,26 @@ def resultValues(model, libname, data, simulator=None): data.get("parsing") or 0.0 ) +def cpu_name(): + if isWin: + return processor() + else: + for line in open("/proc/cpuinfo").readlines(): + if "model name" in line.strip(): + return (re.sub( ".*model name.*:", "", line, count=1)).strip() + +if isWin: + lsb_release = "" +else: + try: + lsb_release = check_output_log(commands + ["cat","/etc/lsb-release"]).decode().strip() + lsb_release = dict(a.split("=") for a in lsb_release.split("\n"))["DISTRIB_DESCRIPTION"].strip('"') + except: + lsb_release = "" + +hostname = socket.gethostname() +sysInfo = "%s: %s, %d GB RAM, %s%s" % (hostname, cpu_name(), int(math.ceil(psutil.virtual_memory().total / (1024.0**3))), ("Docker " + docker + " ") if docker else "", lsb_release) + for (resultBranch, simulator) in resultBranches: db.createTables(resultBranch) for key in stats.keys(): @@ -1003,7 +1024,7 @@ def resultValues(model, libname, data, simulator=None): resultValues(model, libname, data, simulator)) for libname in stats_by_libname.keys(): confighash = stats_by_libname[libname]["conf"]["confighash"] - cursor.execute("INSERT INTO libversion VALUES (?,?,?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash)) + cursor.execute("INSERT INTO libversion VALUES (?,?,?,?,?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash, hostname, sysInfo)) cursor.execute("INSERT INTO omcversion VALUES (?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, omc_version)) """ # Not really a good thing to do; was just done to make generation of the report simpler @@ -1028,25 +1049,6 @@ def checkPhase(phase, n): def is_non_zero_file(fpath): return os.path.isfile(os.path.normpath(fpath)) and os.path.getsize(os.path.normpath(fpath)) > 0 -def cpu_name(): - if isWin: - return processor() - else: - for line in open("/proc/cpuinfo").readlines(): - if "model name" in line.strip(): - return (re.sub( ".*model name.*:", "", line, count=1)).strip() - -if isWin: - lsb_release = "" -else: - try: - lsb_release = check_output_log(commands + ["cat","/etc/lsb-release"]).decode().strip() - lsb_release = dict(a.split("=") for a in lsb_release.split("\n"))["DISTRIB_DESCRIPTION"].strip('"') - except: - lsb_release = "" - -sysInfo = "%s, %d GB RAM, %s%s" % (cpu_name(), int(math.ceil(psutil.virtual_memory().total / (1024.0**3))), ("Docker " + docker + " ") if docker else "", lsb_release) - # create target dir to move results without sync operations (win or when --noSync is used) def stageRootFor(simulator, suffix): """The directory a branch is published from.