From 6a5a100d2f34177b4abe878143a8ae10e8767360 Mon Sep 17 00:00:00 2001 From: rushabh-v Date: Sun, 27 Sep 2020 16:50:51 +0530 Subject: [PATCH 1/2] make py3 compatible --- requirements.txt | 2 +- sqlnet/model/modules/aggregator_predict.py | 6 +-- sqlnet/model/modules/selection_predict.py | 6 +-- .../modules/seq2sql_condition_predict.py | 4 +- .../model/modules/sqlnet_condition_predict.py | 6 +-- sqlnet/model/modules/word_embedding.py | 4 +- sqlnet/model/seq2sql.py | 16 +++---- sqlnet/model/sqlnet.py | 14 +++--- sqlnet/utils.py | 10 ++-- test.py | 34 ++++++------- train.py | 48 +++++++++---------- 11 files changed, 75 insertions(+), 75 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3ed1fc1..67bd4ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ jdcal==1.3 odfpy==1.3.5 olefile==0.44 openpyxl==2.4.9 -pkg-resources==0.0.0 +#pkg-resources pytz==2017.3 records==0.5.2 SQLAlchemy==1.1.14 diff --git a/sqlnet/model/modules/aggregator_predict.py b/sqlnet/model/modules/aggregator_predict.py index 047d2ea..dfc25b2 100644 --- a/sqlnet/model/modules/aggregator_predict.py +++ b/sqlnet/model/modules/aggregator_predict.py @@ -4,7 +4,7 @@ import torch.nn.functional as F from torch.autograd import Variable import numpy as np -from net_utils import run_lstm, col_name_encode +from .net_utils import run_lstm, col_name_encode @@ -17,13 +17,13 @@ def __init__(self, N_word, N_h, N_depth, use_ca): num_layers=N_depth, batch_first=True, dropout=0.3, bidirectional=True) if use_ca: - print "Using column attention on aggregator predicting" + print("Using column attention on aggregator predicting") self.agg_col_name_enc = nn.LSTM(input_size=N_word, hidden_size=N_h/2, num_layers=N_depth, batch_first=True, dropout=0.3, bidirectional=True) self.agg_att = nn.Linear(N_h, N_h) else: - print "Not using column attention on aggregator predicting" + print("Not using column attention on aggregator predicting") self.agg_att = nn.Linear(N_h, 1) self.agg_out = nn.Sequential(nn.Linear(N_h, N_h), nn.Tanh(), nn.Linear(N_h, 6)) diff --git a/sqlnet/model/modules/selection_predict.py b/sqlnet/model/modules/selection_predict.py index 73d8a6a..fa7949d 100644 --- a/sqlnet/model/modules/selection_predict.py +++ b/sqlnet/model/modules/selection_predict.py @@ -4,7 +4,7 @@ import torch.nn.functional as F from torch.autograd import Variable import numpy as np -from net_utils import run_lstm, col_name_encode +from .net_utils import run_lstm, col_name_encode class SelPredictor(nn.Module): def __init__(self, N_word, N_h, N_depth, max_tok_num, use_ca): @@ -15,10 +15,10 @@ def __init__(self, N_word, N_h, N_depth, max_tok_num, use_ca): num_layers=N_depth, batch_first=True, dropout=0.3, bidirectional=True) if use_ca: - print "Using column attention on selection predicting" + print("Using column attention on selection predicting") self.sel_att = nn.Linear(N_h, N_h) else: - print "Not using column attention on selection predicting" + print ("Not using column attention on selection predicting") self.sel_att = nn.Linear(N_h, 1) self.sel_col_name_enc = nn.LSTM(input_size=N_word, hidden_size=N_h/2, num_layers=N_depth, batch_first=True, diff --git a/sqlnet/model/modules/seq2sql_condition_predict.py b/sqlnet/model/modules/seq2sql_condition_predict.py index 4bb34ba..35c1ab8 100644 --- a/sqlnet/model/modules/seq2sql_condition_predict.py +++ b/sqlnet/model/modules/seq2sql_condition_predict.py @@ -4,12 +4,12 @@ import torch.nn.functional as F from torch.autograd import Variable import numpy as np -from net_utils import run_lstm +from .net_utils import run_lstm class Seq2SQLCondPredictor(nn.Module): def __init__(self, N_word, N_h, N_depth, max_col_num, max_tok_num, gpu): super(Seq2SQLCondPredictor, self).__init__() - print "Seq2SQL where prediction" + print("Seq2SQL where prediction") self.N_h = N_h self.max_tok_num = max_tok_num self.max_col_num = max_col_num diff --git a/sqlnet/model/modules/sqlnet_condition_predict.py b/sqlnet/model/modules/sqlnet_condition_predict.py index 1eb5500..4f87d12 100644 --- a/sqlnet/model/modules/sqlnet_condition_predict.py +++ b/sqlnet/model/modules/sqlnet_condition_predict.py @@ -4,7 +4,7 @@ import torch.nn.functional as F from torch.autograd import Variable import numpy as np -from net_utils import run_lstm, col_name_encode +from .net_utils import run_lstm, col_name_encode class SQLNetCondPredictor(nn.Module): def __init__(self, N_word, N_h, N_depth, max_col_num, max_tok_num, use_ca, gpu): @@ -32,10 +32,10 @@ def __init__(self, N_word, N_h, N_depth, max_col_num, max_tok_num, use_ca, gpu): num_layers=N_depth, batch_first=True, dropout=0.3, bidirectional=True) if use_ca: - print "Using column attention on where predicting" + print("Using column attention on where predicting") self.cond_col_att = nn.Linear(N_h, N_h) else: - print "Not using column attention on where predicting" + print("Not using column attention on where predicting") self.cond_col_att = nn.Linear(N_h, 1) self.cond_col_name_enc = nn.LSTM(input_size=N_word, hidden_size=N_h/2, num_layers=N_depth, batch_first=True, diff --git a/sqlnet/model/modules/word_embedding.py b/sqlnet/model/modules/word_embedding.py index b41a0a7..c3c0b3a 100644 --- a/sqlnet/model/modules/word_embedding.py +++ b/sqlnet/model/modules/word_embedding.py @@ -16,14 +16,14 @@ def __init__(self, word_emb, N_word, gpu, SQL_TOK, self.SQL_TOK = SQL_TOK if trainable: - print "Using trainable embedding" + print("Using trainable embedding") self.w2i, word_emb_val = word_emb self.embedding = nn.Embedding(len(self.w2i), N_word) self.embedding.weight = nn.Parameter( torch.from_numpy(word_emb_val.astype(np.float32))) else: self.word_emb = word_emb - print "Using fixed embedding" + print("Using fixed embedding") def gen_x_batch(self, q, col): diff --git a/sqlnet/model/seq2sql.py b/sqlnet/model/seq2sql.py index 651a4ea..8f5e2a6 100644 --- a/sqlnet/model/seq2sql.py +++ b/sqlnet/model/seq2sql.py @@ -4,10 +4,10 @@ import torch.nn.functional as F from torch.autograd import Variable import numpy as np -from modules.word_embedding import WordEmbedding -from modules.aggregator_predict import AggPredictor -from modules.selection_predict import SelPredictor -from modules.seq2sql_condition_predict import Seq2SQLCondPredictor +from .modules.word_embedding import WordEmbedding +from .modules.aggregator_predict import AggPredictor +from .modules.selection_predict import SelPredictor +from .modules.seq2sql_condition_predict import Seq2SQLCondPredictor # This is a re-implementation based on the following paper: @@ -199,9 +199,9 @@ def reinforce_backward(self, score, rewards): def check_acc(self, vis_info, pred_queries, gt_queries, pred_entry): def pretty_print(vis_data): - print 'question:', vis_data[0] - print 'headers: (%s)'%(' || '.join(vis_data[1])) - print 'query:', vis_data[2] + print('question:', vis_data[0]) + print('headers: (%s)'%(' || '.join(vis_data[1]))) + print('query:', vis_data[2]) def gen_cond_str(conds, header): if len(conds) == 0: @@ -349,7 +349,7 @@ def merge_tokens(tok_list, raw_tok_str): cond_toks.append(cond_val) if verbose: - print cond_toks + print(cond_toks) if len(cond_toks) > 0: cond_toks = cond_toks[1:] st = 0 diff --git a/sqlnet/model/sqlnet.py b/sqlnet/model/sqlnet.py index 7c3ebb4..fbfec9e 100644 --- a/sqlnet/model/sqlnet.py +++ b/sqlnet/model/sqlnet.py @@ -4,10 +4,10 @@ import torch.nn.functional as F from torch.autograd import Variable import numpy as np -from modules.word_embedding import WordEmbedding -from modules.aggregator_predict import AggPredictor -from modules.selection_predict import SelPredictor -from modules.sqlnet_condition_predict import SQLNetCondPredictor +from .modules.word_embedding import WordEmbedding +from .modules.aggregator_predict import AggPredictor +from .modules.selection_predict import SelPredictor +from .modules.sqlnet_condition_predict import SQLNetCondPredictor class SQLNet(nn.Module): @@ -231,9 +231,9 @@ def loss(self, score, truth_num, pred_entry, gt_where): def check_acc(self, vis_info, pred_queries, gt_queries, pred_entry): def pretty_print(vis_data): - print 'question:', vis_data[0] - print 'headers: (%s)'%(' || '.join(vis_data[1])) - print 'query:', vis_data[2] + print('question:', vis_data[0]) + print('headers: (%s)'%(' || '.join(vis_data[1]))) + print('query:', vis_data[2]) def gen_cond_str(conds, header): if len(conds) == 0: diff --git a/sqlnet/utils.py b/sqlnet/utils.py index 2311671..c4a09ac 100644 --- a/sqlnet/utils.py +++ b/sqlnet/utils.py @@ -1,5 +1,5 @@ import json -from lib.dbengine import DBEngine +from .lib.dbengine import DBEngine import re import numpy as np #from nltk.tokenize import StanfordTokenizer @@ -14,7 +14,7 @@ def load_data(sql_paths, table_paths, use_small=False): max_col_num = 0 for SQL_PATH in sql_paths: - print "Loading data from %s"%SQL_PATH + print("Loading data from %s"%SQL_PATH) with open(SQL_PATH) as inf: for idx, line in enumerate(inf): if use_small and idx >= 1000: @@ -23,7 +23,7 @@ def load_data(sql_paths, table_paths, use_small=False): sql_data.append(sql) for TABLE_PATH in table_paths: - print "Loading data from %s"%TABLE_PATH + print("Loading data from %s"%TABLE_PATH) with open(TABLE_PATH) as inf: for line in inf: tab = json.loads(line.strip()) @@ -36,7 +36,7 @@ def load_data(sql_paths, table_paths, use_small=False): def load_dataset(dataset_id, use_small=False): if dataset_id == 0: - print "Loading from original dataset" + print("Loading from original dataset") sql_data, table_data = load_data('data/train_tok.jsonl', 'data/train_tok.tables.jsonl', use_small=use_small) val_sql_data, val_table_data = load_data('data/dev_tok.jsonl', @@ -47,7 +47,7 @@ def load_dataset(dataset_id, use_small=False): DEV_DB = 'data/dev.db' TEST_DB = 'data/test.db' else: - print "Loading from re-split dataset" + print("Loading from re-split dataset") sql_data, table_data = load_data('data_resplit/train.jsonl', 'data_resplit/tables.jsonl', use_small=use_small) val_sql_data, val_table_data = load_data('data_resplit/dev.jsonl', diff --git a/test.py b/test.py index 52dbff3..f37576a 100644 --- a/test.py +++ b/test.py @@ -52,32 +52,32 @@ if args.train_emb: agg_m, sel_m, cond_m, agg_e, sel_e, cond_e = best_model_name(args) - print "Loading from %s"%agg_m + print("Loading from %s"%agg_m) model.agg_pred.load_state_dict(torch.load(agg_m)) - print "Loading from %s"%sel_m + print("Loading from %s"%sel_m) model.sel_pred.load_state_dict(torch.load(sel_m)) - print "Loading from %s"%cond_m + print("Loading from %s"%cond_m) model.cond_pred.load_state_dict(torch.load(cond_m)) - print "Loading from %s"%agg_e + print("Loading from %s"%agg_e) model.agg_embed_layer.load_state_dict(torch.load(agg_e)) - print "Loading from %s"%sel_e + print("Loading from %s"%sel_e) model.sel_embed_layer.load_state_dict(torch.load(sel_e)) - print "Loading from %s"%cond_e + print("Loading from %s"%cond_e) model.cond_embed_layer.load_state_dict(torch.load(cond_e)) else: agg_m, sel_m, cond_m = best_model_name(args) - print "Loading from %s"%agg_m + print("Loading from %s"%agg_m) model.agg_pred.load_state_dict(torch.load(agg_m)) - print "Loading from %s"%sel_m + print("Loading from %s"%sel_m) model.sel_pred.load_state_dict(torch.load(sel_m)) - print "Loading from %s"%cond_m + print("Loading from %s"%cond_m) model.cond_pred.load_state_dict(torch.load(cond_m)) - print "Dev acc_qm: %s;\n breakdown on (agg, sel, where): %s"%epoch_acc( - model, BATCH_SIZE, val_sql_data, val_table_data, TEST_ENTRY) - print "Dev execution acc: %s"%epoch_exec_acc( - model, BATCH_SIZE, val_sql_data, val_table_data, DEV_DB) - print "Test acc_qm: %s;\n breakdown on (agg, sel, where): %s"%epoch_acc( - model, BATCH_SIZE, test_sql_data, test_table_data, TEST_ENTRY) - print "Test execution acc: %s"%epoch_exec_acc( - model, BATCH_SIZE, test_sql_data, test_table_data, TEST_DB) + print("Dev acc_qm: %s;\n breakdown on (agg, sel, where): %s"%epoch_acc( + model, BATCH_SIZE, val_sql_data, val_table_data, TEST_ENTRY)) + print("Dev execution acc: %s"%epoch_exec_acc( + model, BATCH_SIZE, val_sql_data, val_table_data, DEV_DB)) + print("Test acc_qm: %s;\n breakdown on (agg, sel, where): %s"%epoch_acc( + model, BATCH_SIZE, test_sql_data, test_table_data, TEST_ENTRY)) + print("Test execution acc: %s"%epoch_exec_acc( + model, BATCH_SIZE, test_sql_data, test_table_data, TEST_DB)) diff --git a/train.py b/train.py index ed0cab5..8992b76 100644 --- a/train.py +++ b/train.py @@ -66,38 +66,38 @@ if args.rl or args.train_emb: # Load pretrained model. agg_lm, sel_lm, cond_lm = best_model_name(args, for_load=True) - print "Loading from %s"%agg_lm + print ("Loading from %s"%agg_lm) model.agg_pred.load_state_dict(torch.load(agg_lm)) - print "Loading from %s"%sel_lm + print ("Loading from %s"%sel_lm) model.sel_pred.load_state_dict(torch.load(sel_lm)) - print "Loading from %s"%cond_lm + print ("Loading from %s"%cond_lm) model.cond_pred.load_state_dict(torch.load(cond_lm)) if args.rl: best_acc = 0.0 best_idx = -1 - print "Init dev acc_qm: %s\n breakdown on (agg, sel, where): %s"% \ + print("Init dev acc_qm: %s\n breakdown on (agg, sel, where): %s"% \ epoch_acc(model, BATCH_SIZE, val_sql_data,\ - val_table_data, TRAIN_ENTRY) - print "Init dev acc_ex: %s"%epoch_exec_acc( - model, BATCH_SIZE, val_sql_data, val_table_data, DEV_DB) + val_table_data, TRAIN_ENTRY)) + print ("Init dev acc_ex: %s"%epoch_exec_acc( + model, BATCH_SIZE, val_sql_data, val_table_data, DEV_DB)) torch.save(model.cond_pred.state_dict(), cond_m) for i in range(100): - print 'Epoch %d @ %s'%(i+1, datetime.datetime.now()) - print ' Avg reward = %s'%epoch_reinforce_train( - model, optimizer, BATCH_SIZE, sql_data, table_data, TRAIN_DB) - print ' dev acc_qm: %s\n breakdown result: %s'% epoch_acc( - model, BATCH_SIZE, val_sql_data, val_table_data, TRAIN_ENTRY) + print ('Epoch %d @ %s'%(i+1, datetime.datetime.now())) + print (' Avg reward = %s'%epoch_reinforce_train( + model, optimizer, BATCH_SIZE, sql_data, table_data, TRAIN_DB)) + print (' dev acc_qm: %s\n breakdown result: %s'% epoch_acc( + model, BATCH_SIZE, val_sql_data, val_table_data, TRAIN_ENTRY)) exec_acc = epoch_exec_acc( model, BATCH_SIZE, val_sql_data, val_table_data, DEV_DB) - print ' dev acc_ex: %s', exec_acc + print (' dev acc_ex: %s', exec_acc) if exec_acc[0] > best_acc: best_acc = exec_acc[0] best_idx = i+1 torch.save(model.cond_pred.state_dict(), 'saved_model/epoch%d.cond_model%s'%(i+1, args.suffix)) torch.save(model.cond_pred.state_dict(), cond_m) - print ' Best exec acc = %s, on epoch %s'%(best_acc, best_idx) + print (' Best exec acc = %s, on epoch %s'%(best_acc, best_idx)) else: init_acc = epoch_acc(model, BATCH_SIZE, val_sql_data, val_table_data, TRAIN_ENTRY) @@ -107,8 +107,8 @@ best_sel_idx = 0 best_cond_acc = init_acc[1][2] best_cond_idx = 0 - print 'Init dev acc_qm: %s\n breakdown on (agg, sel, where): %s'%\ - init_acc + print ('Init dev acc_qm: %s\n breakdown on (agg, sel, where): %s'%\ + init_acc) if TRAIN_AGG: torch.save(model.agg_pred.state_dict(), agg_m) if args.train_emb: @@ -122,16 +122,16 @@ if args.train_emb: torch.save(model.cond_embed_layer.state_dict(), cond_e) for i in range(100): - print 'Epoch %d @ %s'%(i+1, datetime.datetime.now()) - print ' Loss = %s'%epoch_train( + print('Epoch %d @ %s'%(i+1, datetime.datetime.now())) + print(' Loss = %s'%epoch_train( model, optimizer, BATCH_SIZE, - sql_data, table_data, TRAIN_ENTRY) - print ' Train acc_qm: %s\n breakdown result: %s'%epoch_acc( - model, BATCH_SIZE, sql_data, table_data, TRAIN_ENTRY) + sql_data, table_data, TRAIN_ENTRY)) + print(' Train acc_qm: %s\n breakdown result: %s'%epoch_acc( + model, BATCH_SIZE, sql_data, table_data, TRAIN_ENTRY)) #val_acc = epoch_token_acc(model, BATCH_SIZE, val_sql_data, val_table_data, TRAIN_ENTRY) val_acc = epoch_acc(model, BATCH_SIZE, val_sql_data, val_table_data, TRAIN_ENTRY) - print ' Dev acc_qm: %s\n breakdown result: %s'%val_acc + print(' Dev acc_qm: %s\n breakdown result: %s'%val_acc) if TRAIN_AGG: if val_acc[1][0] > best_agg_acc: best_agg_acc = val_acc[1][0] @@ -165,6 +165,6 @@ torch.save(model.cond_embed_layer.state_dict(), 'saved_model/epoch%d.cond_embed%s'%(i+1, args.suffix)) torch.save(model.cond_embed_layer.state_dict(), cond_e) - print ' Best val acc = %s, on epoch %s individually'%( + print(' Best val acc = %s, on epoch %s individually'%( (best_agg_acc, best_sel_acc, best_cond_acc), - (best_agg_idx, best_sel_idx, best_cond_idx)) + (best_agg_idx, best_sel_idx, best_cond_idx))) From 617a5e8f60e7531871eab8326e649e423dbe4f19 Mon Sep 17 00:00:00 2001 From: rushabh-v Date: Sun, 27 Sep 2020 17:28:44 +0530 Subject: [PATCH 2/2] make py3 changes --- extract_vocab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extract_vocab.py b/extract_vocab.py index 970f488..f16ccb1 100644 --- a/extract_vocab.py +++ b/extract_vocab.py @@ -21,7 +21,7 @@ load_dataset(0, use_small=USE_SMALL) word_emb = load_word_emb('glove/glove.%dB.%dd.txt'%(B_word,N_word), use_small=USE_SMALL) -print "Length of word vocabulary: %d"%len(word_emb) +print("Length of word vocabulary: %d"%len(word_emb)) word_to_idx = {'':0, '':1, '':2} word_num = 3 @@ -57,7 +57,7 @@ def check_and_add(tok): for tok in col: check_and_add(tok) -print "Length of used word vocab: %s"%len(word_to_idx) +print("Length of used word vocab: %s"%len(word_to_idx)) emb_array = np.stack(embs, axis=0) with open('glove/word2idx.json', 'w') as outf: