Skip to content
Snippets Groups Projects
Commit bbfb25bf authored by Craig Tiller's avatar Craig Tiller
Browse files

Sanitize before bigquery upload

parent 56cea8f2
No related branches found
No related tags found
No related merge requests found
...@@ -73,6 +73,14 @@ columns = [ ...@@ -73,6 +73,14 @@ columns = [
('framing_bytes_per_iteration', 'float'), ('framing_bytes_per_iteration', 'float'),
] ]
SANITIZE = {
'integer': int,
'float': float,
'boolean': bool,
'string': str,
'timestamp': str,
}
if sys.argv[1] == '--schema': if sys.argv[1] == '--schema':
print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns) print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns)
sys.exit(0) sys.exit(0)
...@@ -89,7 +97,10 @@ else: ...@@ -89,7 +97,10 @@ else:
writer = csv.DictWriter(sys.stdout, [c for c,t in columns]) writer = csv.DictWriter(sys.stdout, [c for c,t in columns])
for row in bm_json.expand_json(js, js2): for row in bm_json.expand_json(js, js2):
if 'label' in row: sane_row = {}
del row['label'] for name, sql_type in columns:
del row['cpp_name'] if name in row:
writer.writerow(row) if row[name] == '': continue
sane_row[name] = SANITIZE[sql_type](row[name])
writer.writerow(sane_row)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment