Ranking of Moscow districts by rental price with Python

def timer(f):
    def wrap_timer(*args, **kwargs):
        start = time.time()
        result = f(*args, **kwargs)
        delta = time.time() - start
        print (f'   {f.__name__}  {delta} ')
        return result
    return wrap_timer

def log(f):
    def wrap_log(*args, **kwargs):
        logging.info(f"  {f.__doc__}")
        result = f(*args, **kwargs)
        logging.info(f": {result}")
        return result
    return wrap_log
logging.basicConfig(level=logging.INFO)

@timer
@log
def requests_site(N):
    headers = ({'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15'})
    pages = [106 + i for i in range(N)]
    n = 0
    for i in pages:
        s = f"https://www.cian.ru/cat.php?deal_type=rent&engine_version=2&page={i}&offer_type=flat&region=1&room1=1&type=-2"
        response = requests.get(s, headers = headers)
        if response.status_code == 200:
            name = f'sheets/sheet_{i}.txt'
            with open(name, 'w') as f:
                f.write(response.text)
            n += 1
            logging.info(f"  {i}")
        else:
            print(f"  {i}   response.status_code = {response.status_code}")
        time.sleep(np.random.randint(7,13))
    return f"  {n} "
requests_site(300)

from bs4 import BeautifulSoup
import re
import pandas as pd
from dateutil.parser import parse
from datetime import datetime, date, time

def read_file(filename):
    with open(filename) as input_file:
        text = input_file.read()
    return text

import tqdm

site_texts = []
pages = [1 + i for i in range(309)]
        
for i in tqdm.tqdm(pages):
    name = f'sheets/sheet_{i}.txt'
    site_texts.append(read_file(name))
    
print(f" {len(site_texts)} .")

def parse_tag(tag, tag_value, item):
    key = tag
    value = "None"
    if item.find('div', {'class': tag_value}):
        if key == 'link':
            value = item.find('div', {'class': tag_value}).find('a').get('href')
        elif (key == 'price' or key == 'price_meter'):
            value = parse_digits(item.find('div', {'class': tag_value}).text, key)
        elif key == 'pub_datetime':
            value = parse_date(item.find('div', {'class': tag_value}).text)
        else:
            value = item.find('div', {'class': tag_value}).text
    return key, value


def parse_digits(string, type_digit):
    digit = 0
    try:
        if type_digit == 'flats_counts':
            digit = int(re.sub(r" ", "", string[:string.find("")]))
        elif type_digit == 'price':
            digit = re.sub(r" ", "", re.sub(r"₽", "", string))
        elif type_digit == 'price_meter':
            digit = re.sub(r" ", "", re.sub(r"₽/²", "", string))
    except:
        return -1
    return digit

def parse_date(string):
    now = datetime.strptime("15.03.20 00:00", "%d.%m.%y %H:%M")
    s = string
    if string.find('') >= 0:
        s = "{} {}".format(now.day, now.strftime("%b"))
        s = string.replace('', s)
    elif string.find('') >= 0:
        s = "{} {}".format(now.day - 1, now.strftime("%b"))
        s = string.replace('',s) 
    if (s.find('') > 0):
        s = s.replace('','mar')
    if (s.find('') > 0):
        s = s.replace('','feb')
    if (s.find('') > 0):
        s = s.replace('','jan')
    return parse(s).strftime('%Y-%m-%d %H:%M:%S')
    
    
def parse_text(text, index):
    
    tag_table = '_93444fe79c--wrapper--E9jWb'
    tag_items = ['_93444fe79c--card--_yguQ', '_93444fe79c--card--_yguQ']
    tag_flats_counts = '_93444fe79c--totalOffers--22-FL'
    tags = {
        'link':('c6e8ba5398--info-section--Sfnx- c6e8ba5398--main-info--oWcMk','undefined c6e8ba5398--main-info--oWcMk'),
        'desc': ('c6e8ba5398--title--2CW78','c6e8ba5398--single_title--22TGT', 'c6e8ba5398--subtitle--UTwbQ'),
        'price': ('c6e8ba5398--header--1df-X', 'c6e8ba5398--header--1dF9r'),
        'price_meter': 'c6e8ba5398--term--3kvtJ',
        'metro': 'c6e8ba5398--underground-name--1efZ3',
        'pub_datetime': 'c6e8ba5398--absolute--9uFLj',
        'address': 'c6e8ba5398--address-links--1tfGW',
        'square': ''
    }
    
    res = []
    flats_counts = 0
    soup = BeautifulSoup(text)  
    if soup.find('div', {'class': tag_flats_counts}):
        flats_counts = parse_digits(soup.find('div', {'class': tag_flats_counts}).text, 'flats_counts')
   
    flats_list = soup.find('div', {'class': tag_table}) 
    if flats_list:
        items = flats_list.find_all('div', {'class': tag_items})         
        for i, item in enumerate(items):
            
            d = {'index': index}
            index += 1
            for tag in tags.keys():
                tag_value = tags[tag]
                key, value = parse_tag(tag, tag_value, item)
                d[key] = value
            results[index] = d
        
    return flats_counts, index

from IPython.display import clear_output

sum_flats = 0
index = 0
results = {}
for i, text in enumerate(site_texts):
    flats_counts, index = parse_text(text, index)    
    sum_flats = len(results)
    clear_output(wait=True)
    print(f"  {i + 1} flats = {flats_counts},   {sum_flats} ")
print(f" sum_flats ({sum_flats}) = flats_counts({flats_counts})")

df['price_per_month'] = df['price'].str.strip('/.').astype(int) #price_int
new_desc = df["desc"].str.split(",", n = 3, expand = True) 
df["square"]= new_desc[1].str.strip(' ²').astype(int)
df["floor"]= new_desc[2]
new_address = df['address'].str.split(',', n = 3, expand = True)
df['okrug'] = new_address[1].str.strip(" ")
df['price_per_meter'] = (df['price_per_month'] / df['square']).round(2) #price_std

df = df.drop(['index','metro', 'price_meter','link', 'price','desc','address','pub_datetime','floor'], axis='columns')