My asynchronous code works flawlessly when executed in series using await, however, as soon as I attempt to run it in parallel using .then() for better performance, I encounter null reference errors.
The use of puppeteer in the launch method is not the issue here since that part functions correctly. I have included the code snippet just to show that async is being used where necessary. While I suspect the problem lies in my implementation of async and await, I am unsure whether I should switch to returning promises instead of using await or if I should utilize Promise.all instead?
//results is null
for (const token of allTokens) {
let url = "https://bscscan.com/token/" + token.contractAddress;
this.provider.BscScanService.getTokenDataFromWebsite(url).then((results: ScrapeResults) => {
console.log("Scraperesults: ", results);
this.provider.TokenHolderCountRepo.insertNewTokenHolderCount({
tokenId: token.id,
holderCount: results.holderCount,
date: Date.now()
});
this.provider.TokenIconRepo.upsertNewIconSource({
tokenId: token.id,
iconSrc: results.tokenIconSrc
});
console.log("Done inserts");
});
}
//Working
for (const token of allTokens) {
let url = "https://bscscan.com/token/" + token.contractAddress;
let results:ScrapeResults = await this.provider.BscScanService.getTokenDataFromWebsite(url);
console.log("Scraperesults: ", results);
await this.provider.TokenHolderCountRepo.insertNewTokenHolderCount({
tokenId: token.id,
holderCount: results.holderCount,
date: Date.now()
});
await this.provider.TokenIconRepo.upsertNewIconSource({
tokenId: token.id,
iconSrc: results.tokenIconSrc
});
console.log("Done inserts");
}
Other methods
public async getTokenDataFromWebsite(url: string): Promise<ScrapeResults>{
return await this.scrapingService.getTrackerData(url);
}
public async getTrackerData(url: string): Promise<ScrapeResults> {
let scrapeResult: ScrapeResults;
try
{
scrapeResult = await this.scraper.launch(url);
}
catch(err){
return null;
}
return scrapeResult;
}
async launch(url: string): Promise<ScrapeResults>{
console.log("Accessing URL: " + url);
puppeteer.use(StealthPlugin());
console.log("1");
const browser = await puppeteer.launch({
//@ts-ignore
headless: true,
args: ['--no-sandbox','--disable-setuid-sandbox']
})
console.log("2");
const userAgent = new UserAgent();
console.log("3");
let page = await browser.newPage();
page.setDefaultNavigationTimeout(0);
console.log("4");
await page.goto(url);
await page.setUserAgent(userAgent.toString());
await page.waitForSelector('body');
let pageTitle = await page.title();
console.log("5");
console.log("Start eval");
let holderCount: string[] = await page.$$eval(
'#ContentPlaceHolder1_tr_tokenHolders > div.row.align-items-center > div.col-md-8 > div.d-flex.align-items-center > div.mr-3',
elements => elements.map(element => element.textContent));
let iconSource: string[] = await page.$$eval('img.u-sm-avatar.mr-2[src]', elements => elements.map(img => img.getAttribute('src')));
let tokenName: string[] = await page.$$eval('div.media-body > span.text-secondary.small', elements => elements.map(element => element.textContent))
const images = await page.evaluate(() => Array.from(document.images, e => e.src));
await browser.close();
console.log("End eval");
console.log("Holdercount:", holderCount);
console.log("iconSource:", iconSource);
console.log("tokenName:", tokenName);
return {
holderCount: this.getHolderCountFromWebsiteString(holderCount[0]),
tokenIconSrc: iconSource[0],
tokenName: tokenName[0]
};
}