Saturday, April 9, 2016

Python: Word Frequency Counter

#https://www.youtube.com/watch?v=ZxiJ92-4Qys&index=35&list=PL6gx4Cwl9DGAcbMi1sH6oAMk4JHw91mC_#https://www.youtube.com/watch?v=up5Xehmtn2E&index=36&list=PL6gx4Cwl9DGAcbMi1sH6oAMk4JHw91mC_#https://www.youtube.com/watch?v=E3T5cm0yys0&index=37&list=PL6gx4Cwl9DGAcbMi1sH6oAMk4JHw91mC_
import requests
from bs4 import BeautifulSoup
import operator


def start(url):
    word_list = []
    source_code = requests.get(url).text #lataa lähdekoodin sivustolta    soup = BeautifulSoup(source_code)
    for post_text in soup.findAll('a', {'class': 'TÄHÄN KIRJOITETAAN SE LUOKKA JONKA ASIOITA HALUTAAN OTTAA TALTEEN'}):
        content = post_text.string
        words = content.lower().split()
        for each_word in words:
            print(each_word)
            word_list.append(each_word)
    #clean_up_list(word_list)
def clean_up_list(word_list):
    clean_word_list = []
    for word in word_list:
        symbols = '!"#€%&/()=?`<>,.-;:_¨ˆ' #nämä sympolit poistetaan        for i in range(0, len(symbols)):
            word = word.replace(symbols[i], '')
        if len(word) > 0:
            print(word)
            clean_word_list.append(word)

start('Tähän sivuston osoite')

No comments:

Post a Comment