async crawlList(listURL: string, crawlerDto: CrawlerDto) { if (crawlerDto.from === CrawlerArticleFrom.ArticleFromZhihu) { return await this.crawlZhihuList(listURL, crawlerDto); } if (crawlerDto.from === CrawlerArticleFrom.ArticleFromSegmentJuejin) { return await this.crawlJuejinList(listURL, crawlerDto); } const pageResult = await axios.get(listURL); const $ = cheerio.load(pageResult.data); const articleItems = $(crawlerDto.itemSelector); const articleURLArr: string[] = []; if (articleItems && articleItems.length) { // tslint:disable-next-line: prefer-for-of for (let i = 0; i < articleItems.length; i++) { const articleLink = $(articleItems[i]).find(crawlerDto.itemTitleSelector); let href: string = articleLink.attr('href') || ''; href = href.trim(); if (href) { href = url.resolve(listURL, href); articleURLArr.push(href); } } } await bluebird.map(articleURLArr, (articleURL) => { return this.crawlPage(articleURL, crawlerDto); }, { concurrency: 10 }); }
img: 'http://www.xiaoliaoba.cn' + $(this) .children('.cont-list-main') .find('img') .attr('data-src')
it('renders products', () => { const store = mockStore({ products: [{ "id": 1, "name": "Product1", "price": 299, "image": "" }] }); const wrapper = render(<ConnectedProductList store={store} />); expect(wrapper.find(".product").length).toBe(1); });
function conditionParse(body) { const $ = cheerio.load(body); let condition = $('div.ak-container.ak-panel').eq(0).find('div.ak-list-element').find('div.ak-title').remove('br').text().trim(); condition = sanatizer(condition); let conditionTab = condition.split('et'); conditionTab = conditionTab.map(function (string) { return sanatizer(string).trim(); }); item['conditions'] = conditionTab; }
/** * Scrapes skill order from a OP.GG page * @param {cheerio} $ - The cheerio object */ scrapeSkillOrder($) { let skillorder = ''; const skills = $('.champion-stats__list').eq(2).find('li:not(.champion-stats__list__arrow) > img').each(function(index) { skillorder += (skillorder !== '' ? ' => ' : '') + i18n.__('key-' + ($(this).siblings().text())); }); return skillorder; }
/** * Scrapes skill order from a League of Graphs page * @param {cheerio} $ - The cheerio object */ scrapeSkillOrder($) { return $('h3:contains(Skill Orders)').parent().find('.championSpellLetter').toArray().map(x => i18n.__('key-' + $(x).text().trim())).join(' => '); }
$('#bilingual ul li').find('p').each(function (i, elm) { if ($(this).attr('class') !== 'example-via') { sentenceSample += $(this).text().trim() + '\n' } })
_scrape(html, champion, position, gameMode) { let $ = cheerio.load(html), availablePositions = []; if (gameMode === 'CLASSIC') { $('a[href^="/5v5/champion/"] > table').each(function(index) { if ($(this).css('box-shadow') === 'none') availablePositions.push({ metasrcPosition: $(this).find('h1').text().toLowerCase(), position: $(this).find('h1').text() === 'MID' ? 'MIDDLE' : $(this).find('h1').text() }); else position = $(this).find('h1').text() === 'MID' ? 'MIDDLE' : $(this).find('h1').text(); }); } else position = gameMode; let itemsets = this.scrapeItemSets($, champion, position, this.scrapeSkillOrder($)); return { position, perks: this.scrapePerks($), summonerspells: this.scrapeSummonerSpells($), itemsets, availablePositions, gameMode }; }
/** * Scrapes skill order from a OP.GG page * @param {cheerio} $ - The cheerio object */ scrapeSkillOrder($) { let skillorder = ''; const skills = $('.champion-stats__list').eq(2).find('li:not(.champion-stats__list__arrow) > img').each(function(index) { skillorder += (skillorder !== '' ? ' => ' : '') + i18n.__('key-' + ($(this).siblings().text())); }); return skillorder; }