Skip to content

feat: added multi-provider support and a zero balance ignore config option #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat: added multi-provider support and a zero balance ignore config option #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,5 +91,7 @@ how.js
snapshot.config.json
tx
balances
.cache
.vscode
.vscode

.idea
erc20-snapshot.iml
1 change: 1 addition & 0 deletions .nvmrc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
lts/iron
17 changes: 12 additions & 5 deletions contract.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,17 @@ const Web3 = require("web3");
const Config = require("./config").getConfig();
const Parameters = require("./parameters").get();

const web3 = new Web3(new Web3.providers.HttpProvider((Config || {}).provider || "http://localhost:8545"));
const web3 = [];
let web3Index = 0;
const getWeb3 = () => {
if (web3Index >= web3.length) web3Index = 0;
return web3[web3Index++];
};

(Config || {}).providers.forEach(config =>
web3.push(new Web3(new Web3.providers.HttpProvider(config || "http://localhost:8545")))
);

const contractAddress = (Config || {}).contractAddress;

module.exports.getContract = () => {
const contract = web3.eth.Contract(Parameters.abi, contractAddress);
return contract;
};
module.exports.getContract = () => getWeb3().eth.Contract(Parameters.abi, contractAddress);
5 changes: 3 additions & 2 deletions events/block-by-block.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"use strict";

const FileHelper = require("../file-helper");
const { getContract } = require("../contract");
const Parameters = require("../parameters").get();

const range = (start, end) => {
Expand All @@ -9,15 +10,15 @@ const range = (start, end) => {
.map((_, idx) => start + idx);
};

module.exports.tryBlockByBlock = async (contract, start, end, symbol) => {
module.exports.tryBlockByBlock = async (start, end, symbol) => {
const blocks = range(start, end);

let counter = 0;
for await (const i of blocks) {
counter++;
console.log("%d% Block %d of %d", Math.floor((counter / (end - start)) * 100), i, end);

const pastEvents = await contract.getPastEvents("Transfer", { fromBlock: i, toBlock: i });
const pastEvents = await getContract().getPastEvents("Transfer", { fromBlock: i, toBlock: i });

if (pastEvents.length) {
console.info("Successfully imported ", pastEvents.length, " events");
Expand Down
57 changes: 31 additions & 26 deletions events/blockchain.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ const Web3 = require("web3");
const BlockByBlock = require("./block-by-block");
const BlockReader = require("./block-reader");
const Config = require("../config").getConfig();
const Contract = require("../contract").getContract();
const { getContract } = require("../contract");
const FileHelper = require("../file-helper");
const LastDownloadedBlock = require("./last-downloaded-block");
const Parameters = require("../parameters").get();
Expand All @@ -14,11 +14,21 @@ const { promisify } = require("util");

const sleep = promisify(setTimeout);

const web3 = new Web3(new Web3.providers.HttpProvider((Config || {}).provider || "http://localhost:8545"));
const web3 = [];
let web3Index = 0;
const getWeb3 = () => {
if (web3Index >= web3.length) web3Index = 0;
return web3[web3Index++];
};


(Config || {}).providers.forEach(config =>
web3.push(new Web3(new Web3.providers.HttpProvider(config || "http://localhost:8545")))
);

const groupBy = (objectArray, property) => {
return objectArray.reduce((acc, obj) => {
var key = obj[property];
const key = obj[property];
if (!acc[key]) {
acc[key] = [];
}
Expand All @@ -29,7 +39,8 @@ const groupBy = (objectArray, property) => {

const tryGetEvents = async (start, end, symbol) => {
try {
const pastEvents = await Contract.getPastEvents("Transfer", { fromBlock: start, toBlock: end });
console.log(`Fetching ${end - start} blocks (${start} to ${end})`);
const pastEvents = await getContract().getPastEvents("Transfer", { fromBlock: start, toBlock: end });

if (pastEvents.length) {
console.info("Successfully imported ", pastEvents.length, " events");
Expand All @@ -48,20 +59,20 @@ const tryGetEvents = async (start, end, symbol) => {
}
}
} catch (e) {
console.log("Could not get events due to an error. Now checking block by block.");
await BlockByBlock.tryBlockByBlock(Contract, start, end, symbol);
console.log("Could not get events due to an error. Now checking block by block.", "");
await BlockByBlock.tryBlockByBlock(start, end, symbol);
}
};

module.exports.get = async () => {
const name = await Contract.methods.name().call();
const symbol = await Contract.methods.symbol().call();
const decimals = await Contract.methods.decimals().call();
const blockHeight = await web3.eth.getBlockNumber();
var fromBlock = parseInt(Config.fromBlock) || 0;
const name = await getContract().methods.name().call();
const symbol = await getContract().methods.symbol().call();
const decimals = await getContract().methods.decimals().call();
const blockHeight = await getWeb3().eth.getBlockNumber();
let fromBlock = parseInt(Config.fromBlock) || 0;
const blocksPerBatch = parseInt(Config.blocksPerBatch) || 0;
const delay = parseInt(Config.delay) || 0;
const toBlock = blockHeight;
const toBlock = parseInt(Config.toBlock) || blockHeight;

const lastDownloadedBlock = await LastDownloadedBlock.get(symbol);

Expand All @@ -70,39 +81,33 @@ module.exports.get = async () => {
fromBlock = lastDownloadedBlock + 1;
}

console.log("From %d to %d", fromBlock, toBlock);
console.log("From %d to %d", fromBlock, toBlock, Math.ceil((toBlock - fromBlock) / blocksPerBatch));

let start = fromBlock;
let end = fromBlock + blocksPerBatch;
let end = Math.min(fromBlock + blocksPerBatch, toBlock);
let i = 0;

while (end < toBlock) {
while (start <= toBlock) {
i++;

if (delay) {
await sleep(delay);
}

console.log("Batch", i + 1, " From", start, "to", end);
console.log("Batch", i, " From", start, "to", end);

await tryGetEvents(start, end, symbol);
await tryGetEvents(start, Math.min(end, toBlock), symbol);

start = end + 1;
end = start + blocksPerBatch;

if (end > toBlock) {
end = toBlock;
}
end = Math.min(start + blocksPerBatch, toBlock);
}

const events = await BlockReader.getEvents(symbol);

const data = {
return {
name,
symbol,
decimals,
events: events
events
};

return data;
};
15 changes: 13 additions & 2 deletions export.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,30 @@ const path = require("path");
const FileHelper = require("./file-helper");
const Parameters = require("./parameters").get();
const WalletType = require("./wallet-type");
const Config = require("./config").getConfig();

const objectToCsv = require("csv-writer").createObjectCsvWriter;

module.exports.exportBalances = async (symbol, balances, format) => {
const withType = await WalletType.addType(balances);
let withType = balances;

const checkContract = Config.checkIfContract.toLowerCase() === "yes";
if (checkContract) {
withType = await WalletType.addType(balances);
}

const writeCsv = () => {
const file = Parameters.outputFileNameCSV.replace(/{token}/g, symbol);
FileHelper.ensureDirectory(path.dirname(file));

const header = [{ id: "wallet", title: "holders" }, { id: "balance", title: "amounts" }];

if (checkContract) {
header.push({ id: "type", title: "Type" });
}
const writer = objectToCsv({
path: file,
header: [{ id: "wallet", title: "Wallet" }, { id: "balance", title: "Balance" }, { id: "type", title: "Type" }]
header
});

console.log("Exporting CSV");
Expand Down
23 changes: 21 additions & 2 deletions index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,35 @@ const Balances = require("./balances");
const Config = require("./config");
const Events = require("./events/blockchain");
const Export = require("./export");
const BigNumber = require("bignumber.js");
const { getContract } = require("./contract");

const start = async () => {
await Config.checkConfig();
const format = Config.getConfig().format;
const result = await Events.get();

console.log("Calculating balances of %s (%s)", result.name, result.symbol);
const balances = await Balances.createBalances(result);
let balances = await Balances.createBalances(result);

console.log("Exporting balances");
const totalBalances = balances.reduce((sum, balance) =>
new BigNumber(balance.balance).plus(sum)
,
0);

if (Config.getConfig().ignoreZeroBalances.toLowerCase() === "yes") {
balances = balances.filter(balance =>
!new BigNumber(balance.balance).isZero()
);
}

// double check 100 first balances
for await (const balance of balances.slice(0, 100)) {
const realBalance = await getContract().methods.balanceOf(balance.wallet).call();
if (!(new BigNumber(realBalance?._hex).div(10 ** parseInt(result.decimals)).eq(balance.balance))) console.warn(`Invalid balance`, balance.wallet, balance.balance);
}

console.log("Exporting balances", `${balances.length} wallets`, `${totalBalances.toString()} ${result.symbol}`);
await Export.exportBalances(result.symbol, balances, format);
};

Expand Down
Loading