嘿，兄die，我是小右。交个朋友吧～

CoinMarketCap

需求描述

网址: coinmarketcap.com/historical/

将每个日期打开的的历史数据table抓取并输出到Excel

日期数据页: coinmarketcap.com/historical/…

功能实现

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time        : 2022/12/7 22:31
# @Author      : Mini-Right
# @Email       : www@anyu.wang
# @File        : coin_market_cap.py
# @Software    : PyCharm
# @Description :
import datetime
from typing import List

import arrow
import requests
import xlsxwriter
from requests.adapters import HTTPAdapter

"""
    网址: https://coinmarketcap.com/historical/
    起始日期: 20130428
    间隔时间: 7 day (sun day)

    数据获取API: https://api.coinmarketcap.com/data-api/v3/cryptocurrency/listings/historical?convertId=2781,1&date=2022-12-04&limit=200&start=1

"""


class CoinMarketCap(object):

    def __init__(self):
        self.session = requests.session()
        self.session.mount('https://', HTTPAdapter(max_retries=3))

    def historical_api(self, date: str, start: int = 1, limit: int = 200):
        """
        获取历史数据接口
        :param date:    日期  格式: YYYY-MM-DD
        :param start:   起始条数    默认: 1
        :param limit:   获取条数    默认: 200
        :return:
        """
        # time.sleep(1)
        url = "https://api.coinmarketcap.com/data-api/v3/cryptocurrency/listings/historical"
        params = {
            "convertId": "2781,1",
            "date": date,
            "limit": str(limit),
            "start": str(start)
        }
        headers = {
            "accept": "application/json, text/plain, */*",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q\u003d0.9,en;q\u003d0.8",
            "cache-control": "no-cache",
            "origin": "https://coinmarketcap.com",
            "platform": "web",
            "referer": "https://coinmarketcap.com/",
            "sec-ch-ua": "\"Google Chrome\";v\u003d\"107\", \"Chromium\";v\u003d\"107\", \"Not\u003dA?Brand\";v\u003d\"24\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"macOS\"",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "same-site",
            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
            "x-request-id": "9f5c086e1a844c29bdf665c050a4bebe",
        }
        response = self.session.get(url=url, params=params, headers=headers)
        if response.status_code == 200:
            return response.json()

    def main(self, date: str):
        start, limit = 1, 200
        excel = ToExcel(excel_path=f"./{date}.xlsx", sheet_name='sheet')
        excel.write_title(['Rank', 'Name', 'Symbol', 'Market Cap', 'Price', 'Circulating Supply', 'volume (24h)', '% 1h', '% 24h', '% 7d'])
        count = 0
        while True:
            # 接口中无分页字段 以start起始条数开始获取  当获取数组为空时 代表这个日期数据全部获取完成
            historical_data: dict = self.historical_api(date=date, start=start)
            item_list: List[dict] = historical_data.get('data')
            if not item_list:
                break
            for item in item_list:
                cmc_rank = item.get('cmcRank')
                name = item.get('name')
                symbol = item.get('symbol')
                quotes = item.get('quotes')
                market_cap = quotes[0].get('marketCap')
                price = quotes[0].get('price')
                total_supply = item.get('totalSupply')
                volume24h = quotes[0].get('volume24h')
                percent_change_1h = quotes[0].get('percentChange1h')
                percent_change_24h = quotes[0].get('percentChange24h')
                percent_change_7d = quotes[0].get('percentChange7d')
                item_info = {
                    "rank": cmc_rank,
                    "name": name,
                    "symbol": symbol,
                    "market_cap": market_cap,
                    "price": price,
                    "total_supply": total_supply,
                    "volume24h": volume24h,
                    "percent_change_1h": percent_change_1h,
                    "percent_change_24h": percent_change_24h,
                    "percent_change_7d": percent_change_7d,
                }
                excel.write(list(item_info.values()))
                count += 1
                print(f"{date}  {count}  {item_info}")

            start += limit

        excel.close()


class ToExcel(object):
    def __init__(self, excel_path: str, sheet_name: str):
        self.f = xlsxwriter.Workbook(
            excel_path,
            options={
                'strings_to_urls': False,
                'default_date_format': '%Y-%m-%d %H:%M:%S'
            })
        self.ws = self.f.add_worksheet(sheet_name)
        self.count = 0

    def write_title(self, data: list):
        for index, value in enumerate(data):
            self.ws.write_string(0, index, str(value))

    def write(self, data: list):
        self.count += 1
        for index, value in enumerate(data):
            self.ws.write_string(self.count, index, str(value))

    def close(self):
        self.f.close()


def get_any_date(date: str, day: int = 0) -> str:
    _format: str = "YYYY-MM-DD"
    return arrow.get(date, _format).shift(days=day).format(_format)


if __name__ == '__main__':
    # 指定日期执行
    # coin = CoinMarketCap()
    # coin.main('2022-12-04')

    # 全量日期执行
    history_date = '2013-04-28'
    while True:
        # 起始时间固定 死循环+7天获取数据  当生成时间大于当天时退出
        if datetime.datetime.strptime(history_date, '%Y-%m-%d') > datetime.datetime.now():
            break
        print(history_date)
        CoinMarketCap().main(history_date)
        history_date = get_any_date(date=history_date, day=7)

公众号: 小右说