sol 链上抓取

import json import time import copy

import scrapy

from currency_index import utils from currency_index.items import SolanaTokenholdersTop100Item

class SolanaTokenholdersTop100Spider(scrapy.Spider): name = 'solana_tokenholders_top100' allowed_domains = ['solscan.io']

def __init__(self, **kwargs):
    super(SolanaTokenholdersTop100Spider, self).__init__(**kwargs)
    self.env = kwargs.get('env', 'online')
    self.tokenholders_top100 = "https://api.solscan.io/token/holders?token={token}&offset={page}&size=20"
    self.request_solana_supply = "https://api.solscan.io/account?address={address}"
    self.start_page = 0

def start_requests(self):
    for target_list in utils.get_chain_target('bsc', 348816):
        for target in target_list:
            if target['explorer'] and target['blockchain_id'] == 348816 and len(
                    str(target['contract_address'])) > 30:
                yield scrapy.Request(
                    url=self.tokenholders_top100.format(token=target['contract_address'].strip(), page=self.start_page),
                    callback=self.parse,
                    meta={'contract_address': target['contract_address'], 'currency_id': target['currency_id'],
                          'page': copy.deepcopy(self.start_page)})

def parse(self, response):
    data_raws = json.loads(response.body.decode())['data']['result']
    contract_address = response.meta['contract_address']
    currency_id = response.meta['currency_id']
    supply = self.get_supply(contract_address)
    item = SolanaTokenholdersTop100Item()
    page = response.meta['page']
    print('address:{} currency_id:{} supply:{}'.format(contract_address, currency_id, supply))
    holder_list = []
    for row in data_raws:
        row_dict = dict()
        row_dict['holder_rank'] = row['rank']
        row_dict['address'] = row['address']
        row_dict['quantity'] = float(row['amount'] / (10 ** row['decimals']))
        row_dict['percentage'] = round((float(row_dict['quantity']) / supply) * 100, 4)
        row_dict['source_id'] = 3
        row_dict['currency_id'] = currency_id
        row_dict['note'] = ''
        row_dict['value'] = ''
        holder_list.append(row_dict)
    item['holder_list'] = holder_list
    yield item


    page += 20
    print('----------page---------------{}-'.format(page))
    yield scrapy.Request(
        url=self.tokenholders_top100.format(token=contract_address.strip(), page=page),
        callback=self.parse,
        meta={'contract_address': contract_address, 'currency_id': currency_id,
              'page': page})



def get_supply(self, contract_address):
    r = utils.get_solana_supply(self.request_solana_supply.format(address=contract_address))
    supply = int(r['data']['tokenInfo']['supply'])
    decimals = r['data']['tokenInfo']['decimals']
    return supply / (10 ** decimals)