乐土为家

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2020-09-28 13:44:18
# Project: hefan

from pyspider.libs.base_handler import *

class Handler(BaseHandler):
    crawl_config = {
    }

    def on_start(self):
        symbols = ['300001', '300002']  # 要爬的股票代码贴这儿。
        for s in symbols:
            self.crawl(f'http://vip.stock.finance.sina.com.cn/corp/go.php/vCB_Bulletin/stockid/{s}/page_type/ndbg.phtml', callback=self.index_page)

    def index_page(self, response):
        self.crawl(response.doc('.datelist > ul > a').attr.href, callback=self.detail_page)

    @config(priority=2)
    def detail_page(self, response):
        return {
            "url": response.url,
            "title": response.doc('#allbulletin > thead > tr > th').text(),
            "content": response.doc('#content').text(),
        }

import hanlp
import json

tokenizer = hanlp.load('LARGE_ALBERT_BASE')
tagger = hanlp.load(hanlp.pretrained.pos.CTB5_POS_RNN_FASTTEXT_ZH)

pipeline = hanlp.pipeline()\
    .append(hanlp.utils.rules.split_sentence, output_key='sentences')\
    .append(tokenizer, output_key='tokens')\
    .append(tagger, output_key='part_of_speech_tags')

# 1. 筛名词
# NN 普通名词，NR 专有名词
# 2. 名词去重
# 3. ｛"002468":["枸橼酸芬太尼", "枸橼酸舒芬太尼", "注射液"]}
# 4. 查某一名词出现在了几只股票里。

with open('C:\\Users\\-\\Desktop\\hefan\\hanlp\\hefan_300xxx.json', encoding='utf-8') as f1, open('C:\\Users\\-\\Desktop\\hefan\\hanlp\\hefan_300xxx_parsed.txt', 'a', encoding='utf-8') as f2:
    for line in f1:
        code = line[136:142]    # 股票代码
        doc = pipeline(line)
        tokens = doc.to_dict()["tokens"]
        part_of_speech_tags = doc.to_dict()["part_of_speech_tags"]

        tmp_tokens = []
        for i in range(len(tokens)):
            for j in range(len(tokens[i])):
                if(part_of_speech_tags[i][j] in ["NN", "NR"]):
                    tmp_tokens.append(tokens[i][j])

        tokens = list(set(tmp_tokens))  # 去重
        f2.write(f"{code}-"+",".join(tokens)+"\n")