如何快速建立论坛的主题模型，或困扰哪些人患有乳糜泻

步骤1.从论坛收集数据

url_coping = 'https://www.celiac.com/forums/forum/5-coping-with-celiac-disease/'

# Get total number of pages
def get_pages_count(url):
    response = requests.get(url, headers={'User-Agent': UserAgent().chrome})
    soup = BeautifulSoup(response.content, 'html.parser')
    last_page_section = soup.find('li', attrs = {'class':'ipsPagination_last'})
    if (last_page_section):
        count_link = last_page_section.find('a')
        return int(count_link['data-page'])
    else: 
        return 1

coping_pages_count = get_pages_count(url_coping)

# collect pages
def retrieve_pages(pages_count, url):
    pages = []
    for page in range(pages_count):
        response = requests.get('{}page/{}'.format(url, page), headers={'User-Agent': UserAgent().chrome})
        soup = BeautifulSoup(response.content, 'html.parser')
        pages.append(soup)
    return pages

coping_pages = retrieve_pages(coping_pages_count, url_coping)

def collect_post_info(pages):
    posts = []
    for page in pages:
        posts_list_soup = page.find('ol', attrs = {'class': 'ipsDataList'}).findAll('li', attrs = {'class': 'ipsDataItem'})
        for post_soup in posts_list_soup:
            post = {}
            post['id'] = uuid.uuid4()
            # collecting titles and urls
            title_section = post_soup.find('span', attrs = {'class':'ipsType_break ipsContained'})
            if (title_section):
                title_section_a = title_section.find('a')
                post['title'] = title_section_a['title']
                post['url'] = title_section_a['data-ipshover-target']
            # collecting author & last action
            author_section = post_soup.find('div', attrs = {'class':'ipsDataItem_meta'})
            if (author_section):
                author_section_a = post_soup.find('a')
                author_section_time = post_soup.find('time')
                post['author'] = author_section_a['data-ipshover-target']
                post['last_action'] = author_section_time['datetime']
            # collecting stats
            stats_section = post_soup.find('ul', attrs = {'class':'ipsDataItem_stats'})
            if (stats_section):
                stats_section_replies = post_soup.find('span', attrs = {'class':'ipsDataItem_stats_number'})
                if (stats_section_replies):
                    post['replies'] = stats_section_replies.getText()
                stats_section_views = post_soup.find('li', attrs = {'class':'ipsType_light'})
                if (stats_section_views):
                    post['views'] = stats_section_views.find('span', attrs = {'class':'ipsDataItem_stats_number'}).getText()
            posts.append(post)
    return posts

coping_posts_info = collect_post_info(coping_pages)

df_coping = pd.DataFrame(coping_posts_info, 
               columns =['title', 'url', 'author', 'last_action', 'replies', 'views']) 

# format data
df_coping['replies'] = df_coping['replies'].astype(int)
df_coping['views'] = df_coping['views'].apply(lambda x: int(x.replace(',','')))
df_coping.to_csv('celiac_forum_coping.csv', sep=',')

def collect_postpage_details(pages, df):
    comments = []
    for i, page in enumerate(pages):
        articles = page.findAll('article')
        for k, article in enumerate(articles):
            comment = {
                'url': df['url'][i]
            }
            if(k == 0):
                comment['question'] = 1
            else:
                comment['question'] = 0
            # collecting comments
            comment_section = article.find('div', attrs = {'class':'ipsComment_content'})
            if (comment_section):
                comment_section_p = comment_section.find('p')
                if(comment_section_p):
                    comment['comment'] = comment_section_p.getText()
            comment['date'] = comment_section.find('time')['datetime']
            author_section = article.find('strong')
            if (author_section):
                author_section_url = author_section.find('a')
                if (author_section_url):
                    comment['author'] = author_section_url['data-ipshover-target']
            comments.append(comment)
    return comments

coping_data = collect_postpage_details(coping_comments_pages, df_coping)
df_coping_comments.to_csv('celiac_forum_coping_comments_1.csv', sep=',')

步骤2数据分析和主题建模

# Let's get rid of text < 30 words
def filter_text_words(text, min_words = 30):
    text = str(text)
    return len(text.split()) > 30
filtered_comments = filtered_comments[filtered_comments['comment'].apply(filter_text_words)]
comments_only = filtered_comments['comment']
comments_only= comments_only.apply(lambda x: x.lower())
comments_only.head()

stop_words = stopwords.words('english')
def remove_stop_words(tokens):
    new_tokens = []
    for t in tokens:
        token = []
        for word in t:
            if word not in stop_words:
                token.append(word)
        new_tokens.append(token)
    return new_tokens

tokens = remove_stop_words(data_words)

bigram = gensim.models.Phrases(tokens, min_count=5, threshold=100)
bigram_mod = gensim.models.phrases.Phraser(bigram)
bigram_mod.save('bigram_mod.pkl')
bag_of_words = [bigram_mod[w] for w in tokens]
with open('bigrams.pkl', 'wb') as f:
    pickle.dump(bag_of_words, f)

id2word = corpora.Dictionary(bag_of_words)
id2word.save('id2word.pkl')
id2word.filter_extremes(no_below=3, no_above=0.4, keep_n=3*10**6)
corpus = [id2word.doc2bow(text) for text in bag_of_words]

lda_model = gensim.models.ldamodel.LdaModel(
    corpus, 
    id2word=id2word, 
    eval_every=20,
    random_state=42,
    num_topics=30, 
    passes=5
    )
lda_model.save('lda_default_2.pkl')
topics = lda_model.show_topics(num_topics=30, num_words=100, formatted=False)

for t in range(lda_model.num_topics):
    plt.figure(figsize=(15, 10))
    plt.imshow(WordCloud(background_color="white", max_words=100, width=900, height=900, collocations=False)
               .fit_words(dict(topics[t][1])))
    plt.axis("off")
    plt.title("Topic #" + themes_headers[t])
    plt.show()

注意，很多图像

---

«Machine Learning» OTUS.

---