various scripting update

export_citations.py

@@ -20,7 +20,7 @@
 c = sys.argv[1]
 output_dir = os.path.join(data_dir, 'out', c)
 if not os.path.exists(output_dir):
-    os.mkdir(output_dir)
+    os.makedirs(output_dir)
 
 row = conf_df.loc[conf_df['Abbrv'] == c]
 conf_id = list(row['ConfID'])[0]
@@ -33,7 +33,7 @@
     paper_file = os.path.join(data_dir, 'data_txt', 'Papers.txt')
     cmd = "grep " + conf_id +' '+ paper_file + ' > ' + conf_paper_file
     print("{} {}".format(datetime.now(), cmd) )
-    subprocess.call(cmd)
+    os.system(cmd)
 
 df_paper = pd.read_table(conf_paper_file, header=None, 
                          names=['PaperID', 'TitleOrig', 'TitleNorm', 'PubYear', 'PubDate', 
@@ -80,4 +80,4 @@
 with open(cited_file, 'wt') as fh:
     fh.write('\n'.join(['\t'.join(s) for s in cited_records] ))
 
-print(' saved to {} and {} \n\n'.format(citing_file, cited_file))
+print(' saved to {} and {} \n\n'.format(citing_file, cited_file))

paperdb.sql

@@ -1,3 +1,4 @@
-CREATE TABLE paper_pruned(id TEXT PRIMARY KEY, year INTEGER, venueid TEXT);
+CREATE TABLE paper_pruned(id TEXT, year INTEGER, venueid TEXT);
 .separator "," 
-.import ./data_txt/Papers_pruned.txt paper_pruned
+.import ./data_txt/Papers_pruned.txt paper_pruned
+CREATE UNIQUE INDEX id_index ON paper_pruned (id);