import requests
import pymysql.cursors
from bs4 import BeautifulSoup

def get_effect_data(data):
    results = list()
    soup = BeautifulSoup(data, \'html.parser\')
    projects = soup.find_all(\'div\', class_=\'repo-list-item\')
    for project in projects:
        writer_project = project.find(\'a\', attrs={\'class\': \'v-align-middle\'})[\'href\'].strip()
        project_language = project.find(\'div\', attrs={\'class\': \'d-table-cell col-2 text-gray pt-2\'}).get_text().strip()
        project_starts = project.find(\'a\', attrs={\'class\': \'muted-link\'}).get_text().strip()
        update_desc = project.find(\'p\', attrs={\'class\': \'f6 text-gray mb-0 mt-2\'}).get_text().strip()

        result = (writer_project.split(\'/\')[1], writer_project.split(\'/\')[2], project_language, project_starts, update_desc)
        results.append(result)
    return results


def get_response_data(page):
    request_url = \'https://github.com/search\'
    params = {\'o\': \'desc\', \'q\': \'python\', \'s\': \'stars\', \'type\': \'Repositories\', \'p\': page}
    resp = requests.get(request_url, params)
    return resp.text


def insert_datas(data):
    connection = pymysql.connect(host=\'localhost\',
                                 user=\'root\',
                                 password=\'root\',
                                 db=\'test\',
                                 charset=\'utf8mb4\',
                                 cursorclass=pymysql.cursors.DictCursor)
    try:
        with connection.cursor() as cursor:
            sql = \'insert into project_info(project_writer, project_name, project_language, project_starts, update_desc) VALUES (%s, %s, %s, %s, %s)\'
            cursor.executemany(sql, data)
            connection.commit()
    except:
        connection.close()


if __name__ == \'__main__\':
    total_page = 2 # 爬虫数据的总页数
    datas = list()
    for page in range(total_page):
        res_data = get_response_data(page + 1)
        data = get_effect_data(res_data)
        datas += data
    insert_datas(datas)