INNER CODE UNIT · Python

v_cosine

Koziev/NLP_Datasets · Samples/sort_facts_by_LSA_tSNE.py:23

def v_cosine(a, b):
    return np.dot(a,b)/(np.linalg.norm(a)*np.linalg.norm(b))

print('Buidling tf-idf corpus...')
tfidf_corpus = set()

with codecs.open(input_path, 'r', 'utf-8') as rdr:
    for line in rdr:
        phrase = line.strip()
        if len(phrase) > 0:
            tfidf_corpus.add(phrase)

tfidf_corpus = list(tfidf_corpus)
print('{} phrases in tfidf corpus'.format(len(tfidf_corpus)))

print('Fitting LSA...')
vectorizer = TfidfVectorizer(max_features=None, ngram_range=(3, 5), min_df=1, analyzer='char')
svd_model = TruncatedSVD(n_components=LSA_DIMS, algorithm='randomized', n_iter=20, random_state=42)

View source record →