From f014aa755b363a8f8e51d8932ecf2f5886761260 Mon Sep 17 00:00:00 2001 From: Ethan Shen <42264778+nczitzk@users.noreply.github.com> Date: Sun, 26 Mar 2023 00:45:19 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(route):=20=E5=8F=82=E8=80=83=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E6=A0=8F=E7=9B=AE=20(#12168)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(route): 参考消息栏目 * apply suggestions from code review --------- --- docs/traditional-media.md | 30 ++++-- lib/v2/cankaoxiaoxi/index.js | 101 ++++++++---------- lib/v2/cankaoxiaoxi/maintainer.js | 3 +- lib/v2/cankaoxiaoxi/radar.js | 59 ++-------- lib/v2/cankaoxiaoxi/router.js | 3 +- lib/v2/cankaoxiaoxi/templates/description.art | 5 + 6 files changed, 84 insertions(+), 117 deletions(-) create mode 100644 lib/v2/cankaoxiaoxi/templates/description.art diff --git a/docs/traditional-media.md b/docs/traditional-media.md index 0bea91efe65cb7..a33abc5b25eda1 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -632,13 +632,31 @@ Category 列表： ## 参考消息 -### 新闻分类 - - +### 栏目 -| 中国 | 国际 | 军事 | 台海 | 财经 | 科技 | 文化 | -| ---------- | ---------- | ------------- | ----------- | ------------ | --------------- | ------------ | -| china_news | world_news | military_news | taiwan_news | finance_news | technology_news | culture_news | + + +| 栏目 | id | +| -------------- | -------- | +| 第一关注 | diyi | +| 中国 | zhongguo | +| 国际 | gj | +| 观点 | guandian | +| 锐参考 | ruick | +| 体育健康 | tiyujk | +| 科技应用 | kejiyy | +| 文化旅游 | wenhualy | +| 参考漫谈 | cankaomt | +| 研究动态 | yjdt | +| 海外智库 | hwzk | +| 业界信息・观点 | yjxx | +| 海外看中国城市 | hwkzgcs | +| 译名趣谈 | ymymqt | +| 译名发布 | ymymfb | +| 双语汇 | ymsyh | +| 参考视频 | video | +| 军事 | junshi | +| 参考人物 | cankaorw | diff --git a/lib/v2/cankaoxiaoxi/index.js b/lib/v2/cankaoxiaoxi/index.js index ffe22e9660c71d..168ce335f891ac 100644 --- a/lib/v2/cankaoxiaoxi/index.js +++ b/lib/v2/cankaoxiaoxi/index.js @@ -1,78 +1,65 @@ const got = require('@/utils/got'); const timezone = require('@/utils/timezone'); const { parseDate } = require('@/utils/parse-date'); -const cheerio = require('cheerio'); - -const nodes = { - china_news: { - title: '中国', - url: 'https://china.cankaoxiaoxi.com/', - }, - world_news: { - title: '国际', - url: 'https://world.cankaoxiaoxi.com/', - }, - military_news: { - title: '军事', - url: 'https://mil.cankaoxiaoxi.com/', - }, - taiwan_news: { - title: '台海', - url: 'https://tw.cankaoxiaoxi.com/', - }, - finance_news: { - title: '财经', - url: 'https://finance.cankaoxiaoxi.com/', - }, - technology_news: { - title: '科技', - url: 'https://science.cankaoxiaoxi.com/', - }, - culture_news: { - title: '文化', - url: 'https://culture.cankaoxiaoxi.com/', - }, -}; +const { art } = require('@/utils/render'); +const path = require('path'); module.exports = async (ctx) => { - const category = ctx.params.category; + const id = ctx.params.id ?? 'diyi'; + const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 50; - const currentUrl = nodes[category].url; - const response = await got({ + const rootUrl = 'https://china.cankaoxiaoxi.com'; + const listApiUrl = `${rootUrl}/json/channel/${id}/list.json`; + const channelApiUrl = `${rootUrl}/json/channel/${id}.channeljson`; + const currentUrl = `${rootUrl}/#/generalColumns/${id}`; + + const listResponse = await got({ method: 'get', - url: currentUrl, + url: listApiUrl, }); - const $ = cheerio.load(response.data); + const channelResponse = await got({ + method: 'get', + url: channelApiUrl, + }); - const list = $('a', '.toutiao') - .filter(function () { - return $(this).text() !== ''; - }) - .map((_, item) => ({ - link: $(item).attr('href'), - })) - .get(); + let items = listResponse.data.list.slice(0, limit).map((item) => ({ + title: item.data.title, + author: item.data.userName, + category: item.data.channelName, + pubDate: timezone(parseDate(item.data.publishTime), +8), + link: item.data.moVideoPath ? item.data.sourceUrl : `${rootUrl}/json/content/${item.data.url.match(/\/pages\/(.*?)\.html/)[1]}.detailjson`, + video: item.data.moVideoPath, + cover: item.data.mCoverImg, + })); - const items = await Promise.all( - list.map((item) => + items = await Promise.all( + items.map((item) => ctx.cache.tryGet(item.link, async () => { - const detailResponse = await got({ - method: 'get', - url: item.link, - }); - const content = cheerio.load(detailResponse.data); - item.title = content('h1.articleHead').text(); - item.pubDate = timezone(parseDate(content('span#pubtime_baidu').text()), +8); - item.author = content('span#source_baidu').text(); - item.description = content('div.articleText').html(); + if (item.video) { + item.description = art(path.join(__dirname, 'templates/description.art'), { + video: item.video, + cover: item.cover, + }); + } else { + const detailResponse = await got({ + method: 'get', + url: item.link, + }); + + const data = detailResponse.data; + + item.link = `${rootUrl}/#/detailsPage/${id}/${data.id}/1/${data.publishTime.split(' ')[0]}`; + item.description = data.txt; + } + return item; }) ) ); ctx.state.data = { - title: `${nodes[category].title} 参考消息`, + title: `参考消息 - ${channelResponse.data.name}`, link: currentUrl, description: '参考消息', language: 'zh-cn', diff --git a/lib/v2/cankaoxiaoxi/maintainer.js b/lib/v2/cankaoxiaoxi/maintainer.js index 3dcd5ba711ac41..a8604819bf0b69 100644 --- a/lib/v2/cankaoxiaoxi/maintainer.js +++ b/lib/v2/cankaoxiaoxi/maintainer.js @@ -1,3 +1,4 @@ module.exports = { - '/news/:category': ['yuxinliu-alex'], + '/column/:id?': ['yuxinliu-alex', 'nczitzk'], + '/:id?': ['yuxinliu-alex', 'nczitzk'], }; diff --git a/lib/v2/cankaoxiaoxi/radar.js b/lib/v2/cankaoxiaoxi/radar.js index 96912d3e06aa2b..d9f42f21515e8b 100644 --- a/lib/v2/cankaoxiaoxi/radar.js +++ b/lib/v2/cankaoxiaoxi/radar.js @@ -1,60 +1,15 @@ module.exports = { 'cankaoxiaoxi.com': { _name: '参考消息', - china: [ + '.': [ { - title: '中国新闻', - docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi', + title: '栏目', + docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi-lan-mu', source: ['/'], - target: '/cankaoxiaoxi/news/china_news', - }, - ], - culture: [ - { - title: '文化新闻', - docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi', - source: ['/'], - target: '/cankaoxiaoxi/news/culture_news', - }, - ], - finance: [ - { - title: '财经新闻', - docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi', - source: ['/'], - target: '/cankaoxiaoxi/news/finance_news', - }, - ], - mil: [ - { - title: '军事新闻', - docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi', - source: ['/'], - target: '/cankaoxiaoxi/news/military_news', - }, - ], - science: [ - { - title: '科技新闻', - docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi', - source: ['/'], - target: '/cankaoxiaoxi/news/technology_news', - }, - ], - tw: [ - { - title: '台海新闻', - docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi', - source: ['/'], - target: '/cankaoxiaoxi/news/taiwan_news', - }, - ], - world: [ - { - title: '国际新闻', - docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi', - source: ['/'], - target: '/cankaoxiaoxi/news/world_news', + target: (params, url) => { + const urlStr = new URL(url).toString(); + return `/cankaoxiaoxi/column${/\/#\//.test(urlStr) ? `/${urlStr.split('/').pop()}` : ''}`; + }, }, ], }, diff --git a/lib/v2/cankaoxiaoxi/router.js b/lib/v2/cankaoxiaoxi/router.js index a9517720c0300e..ba2b77181c59c2 100644 --- a/lib/v2/cankaoxiaoxi/router.js +++ b/lib/v2/cankaoxiaoxi/router.js @@ -1,3 +1,4 @@ module.exports = function (router) { - router.get('/news/:category', require('./index')); + router.get('/column/:id?', require('./')); + router.get('/:id?', require('./')); }; diff --git a/lib/v2/cankaoxiaoxi/templates/description.art b/lib/v2/cankaoxiaoxi/templates/description.art new file mode 100644 index 00000000000000..22843d6743c7bf --- /dev/null +++ b/lib/v2/cankaoxiaoxi/templates/description.art @@ -0,0 +1,5 @@ +{{ if video }} + +{{ /if }} \ No newline at end of file From 294e690478e20f0d7cff3e12a842d2099abb8b49 Mon Sep 17 00:00:00 2001 From: Tony Date: Sat, 25 Mar 2023 08:32:59 -1100 Subject: [PATCH 2/2] feat(route): modelscope (#12178) --- docs/programming.md | 18 +++++++ lib/v2/modelscope/community.js | 55 ++++++++++++++++++++++ lib/v2/modelscope/datasets.js | 55 ++++++++++++++++++++++ lib/v2/modelscope/maintainer.js | 6 +++ lib/v2/modelscope/models.js | 45 ++++++++++++++++++ lib/v2/modelscope/radar.js | 33 +++++++++++++ lib/v2/modelscope/router.js | 6 +++ lib/v2/modelscope/studios.js | 57 +++++++++++++++++++++++ lib/v2/modelscope/templates/community.art | 11 +++++ lib/v2/modelscope/templates/desc.art | 9 ++++ 10 files changed, 295 insertions(+) create mode 100644 lib/v2/modelscope/community.js create mode 100644 lib/v2/modelscope/datasets.js create mode 100644 lib/v2/modelscope/maintainer.js create mode 100644 lib/v2/modelscope/models.js create mode 100644 lib/v2/modelscope/radar.js create mode 100644 lib/v2/modelscope/router.js create mode 100644 lib/v2/modelscope/studios.js create mode 100644 lib/v2/modelscope/templates/community.art create mode 100644 lib/v2/modelscope/templates/desc.art diff --git a/docs/programming.md b/docs/programming.md index 3c5865f74b253d..69ac0ee26645cb 100644 --- a/docs/programming.md +++ b/docs/programming.md @@ -559,6 +559,24 @@ GitHub 官方也提供了一些 RSS: +## ModelScope 魔搭社区 + +### 数据集 + + + +### 模型库 + + + +### 创空间 + + + +### DevPress 官方社区 + + + ## MySQL ### Release Notes diff --git a/lib/v2/modelscope/community.js b/lib/v2/modelscope/community.js new file mode 100644 index 00000000000000..57f2eee04393e5 --- /dev/null +++ b/lib/v2/modelscope/community.js @@ -0,0 +1,55 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); +const path = require('path'); +const timezone = require('@/utils/timezone'); +const { art } = require('@/utils/render'); +const { parseDate } = require('@/utils/parse-date'); + +module.exports = async (ctx) => { + const baseUrl = 'https://community.modelscope.cn'; + + const { data } = await got.post(`${baseUrl}/v1/namespace_page/article`, { + json: { id: 142373, notInMediaAidList: [], pageNum: 1, pageSize: ctx.query.limit ? parseInt(ctx.query.limit) : 30 }, + }); + + const articles = data.data.content.map((c) => ({ + title: c.content.name, + description: c.content.desc, + author: c.nickname, + link: `${baseUrl}/${c.content.id}.html`, + pubDate: timezone(parseDate(c.content.createdTime), 8), + category: c.content.externalData.tags.map((t) => t.name), + thumb: c.content.thumb, + })); + + const items = await Promise.all( + articles.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data } = await got(item.link); + + const $ = cheerio.load(data); + const initialData = JSON.parse( + $('script') + .text() + .match(/window\.__INITIAL_STATE__\s*=\s*({.*?});/)[1] + ); + + item.description = art(path.join(__dirname, 'templates/community.art'), { + thumb: item.thumb, + quote: item.description, + content: initialData.pageData.detail.ext.content, + }); + + return item; + }) + ) + ); + + ctx.state.data = { + title: 'ModelScope魔搭社区-DevPress官方社区', + description: 'ModelScope魔搭社区 DevPress官方社区-ModelScope旨在打造下一代开源的模型即服务共享平台，为泛AI开发者提供灵活、易用、低成本的一站式模型服务产品，让模型应用更简单。', + image: 'https://g.alicdn.com/sail-web/maas/0.8.10/favicon/128.ico', + link: baseUrl, + item: items, + }; +}; diff --git a/lib/v2/modelscope/datasets.js b/lib/v2/modelscope/datasets.js new file mode 100644 index 00000000000000..7293ad2bda545c --- /dev/null +++ b/lib/v2/modelscope/datasets.js @@ -0,0 +1,55 @@ +const got = require('@/utils/got'); +const md = require('markdown-it')({ + html: true, + linkify: true, +}); +const path = require('path'); +const { art } = require('@/utils/render'); +const { parseDate } = require('@/utils/parse-date'); + +module.exports = async (ctx) => { + const baseUrl = 'https://modelscope.cn'; + const link = `${baseUrl}/datasets`; + + const { data } = await got(`${baseUrl}/api/v1/dolphin/datasets`, { + searchParams: { + PageSize: ctx.query.limit ? parseInt(ctx.query.limit) : 36, + PageNumber: 1, + Target: '', + Sort: 'gmt_modified', + }, + }); + + const datasets = data.Data.map((dataset) => ({ + title: dataset.ChineseName, + description: dataset.Description, + author: dataset.CreatedBy, + link: `${link}/${dataset.Namespace}/${dataset.Name}`, + pubDate: parseDate(dataset.GmtCreate, 'X'), + category: dataset.UserDefineTags.split(','), + slug: `/${dataset.Namespace}/${dataset.Name}`, + })); + + const items = await Promise.all( + datasets.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data } = await got(`${baseUrl}/api/v1/datasets${item.slug}`); + + const content = data.Data.ReadmeContent.replace(/img src="(?!http)(.*?)"/g, `img src="${baseUrl}/api/v1/datasets${item.slug}/repo?Revision=master&FilePath=$1&View=true"`); + item.description = art(path.join(__dirname, 'templates/desc.art'), { + description: item.description, + md: md.render(content), + }); + return item; + }) + ) + ); + + ctx.state.data = { + title: '数据集首页 · 魔搭社区', + description: 'ModelScope——汇聚各领域先进的机器学习模型，提供模型探索体验、推理、训练、部署和应用的一站式服务。在这里，共建模型开源社区，发现、学习、定制和分享心仪的模型。', + image: 'https://g.alicdn.com/sail-web/maas/0.8.10/favicon/128.ico', + link, + item: items, + }; +}; diff --git a/lib/v2/modelscope/maintainer.js b/lib/v2/modelscope/maintainer.js new file mode 100644 index 00000000000000..c2e6add994f368 --- /dev/null +++ b/lib/v2/modelscope/maintainer.js @@ -0,0 +1,6 @@ +module.exports = { + '/community': ['TonyRL'], + '/datasets': ['TonyRL'], + '/models': ['TonyRL'], + '/studios': ['TonyRL'], +}; diff --git a/lib/v2/modelscope/models.js b/lib/v2/modelscope/models.js new file mode 100644 index 00000000000000..1b95ce3fe4854b --- /dev/null +++ b/lib/v2/modelscope/models.js @@ -0,0 +1,45 @@ +const got = require('@/utils/got'); +const md = require('markdown-it')({ + html: true, + linkify: true, +}); +const { parseDate } = require('@/utils/parse-date'); + +module.exports = async (ctx) => { + const baseUrl = 'https://modelscope.cn'; + const link = `${baseUrl}/models`; + + const { data } = await got.put(`${baseUrl}/api/v1/dolphin/models`, { + json: { PageSize: ctx.query.limit ? parseInt(ctx.query.limit) : 36, PageNumber: 1, SortBy: 'GmtModified', Target: '', SingleCriterion: [] }, + }); + + const models = data.Data.Model.Models.map((model) => ({ + title: model.ChineseName, + description: model.Description, + author: model.Organization.FullName, + link: `${link}/${model.Path}/${model.Name}`, + pubDate: parseDate(model.CreatedTime, 'X'), + category: [...new Set([...model.Tasks.map((task) => task.ChineseName), ...model.Tags])], + slug: `/${model.Path}/${model.Name}`, + })); + + const items = await Promise.all( + models.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data } = await got(`${baseUrl}/api/v1/models${item.slug}`); + + const content = data.Data.ReadMeContent.replace(/img src="(?!http)(.*?)"/g, `img src="${baseUrl}/api/v1/models${item.slug}/repo?Revision=master&FilePath=$1&View=true"`); + item.description = md.render(content); + return item; + }) + ) + ); + + ctx.state.data = { + title: '模型库首页 · 魔搭社区', + description: 'ModelScope——汇聚各领域先进的机器学习模型，提供模型探索体验、推理、训练、部署和应用的一站式服务。在这里，共建模型开源社区，发现、学习、定制和分享心仪的模型。', + image: 'https://g.alicdn.com/sail-web/maas/0.8.10/favicon/128.ico', + link, + item: items, + }; +}; diff --git a/lib/v2/modelscope/radar.js b/lib/v2/modelscope/radar.js new file mode 100644 index 00000000000000..3167a3b5ca5362 --- /dev/null +++ b/lib/v2/modelscope/radar.js @@ -0,0 +1,33 @@ +module.exports = { + 'modelscope.cn': { + _name: 'ModelScope 魔搭社区', + '.': [ + { + title: '数据集', + docs: 'https://docs.rsshub.app/programming.html#modelscope-mo-da-she-qu', + source: ['/datasets'], + target: '/modelscope/datasets', + }, + { + title: '模型库', + docs: 'https://docs.rsshub.app/programming.html#modelscope-mo-da-she-qu', + source: ['/models'], + target: '/modelscope/models', + }, + { + title: '创空间', + docs: 'https://docs.rsshub.app/programming.html#modelscope-mo-da-she-qu', + source: ['/studios'], + target: '/modelscope/studios', + }, + ], + community: [ + { + title: 'DevPress 官方社区', + docs: 'https://docs.rsshub.app/programming.html#modelscope-mo-da-she-qu', + source: ['/'], + target: '/modelscope/community', + }, + ], + }, +}; diff --git a/lib/v2/modelscope/router.js b/lib/v2/modelscope/router.js new file mode 100644 index 00000000000000..889d162a17ca36 --- /dev/null +++ b/lib/v2/modelscope/router.js @@ -0,0 +1,6 @@ +module.exports = (router) => { + router.get('/community', require('./community')); + router.get('/datasets', require('./datasets')); + router.get('/models', require('./models')); + router.get('/studios', require('./studios')); +}; diff --git a/lib/v2/modelscope/studios.js b/lib/v2/modelscope/studios.js new file mode 100644 index 00000000000000..ac9e67c81d8be2 --- /dev/null +++ b/lib/v2/modelscope/studios.js @@ -0,0 +1,57 @@ +const got = require('@/utils/got'); +const md = require('markdown-it')({ + html: true, + linkify: true, +}); +const path = require('path'); +const { art } = require('@/utils/render'); +const { parseDate } = require('@/utils/parse-date'); + +module.exports = async (ctx) => { + const baseUrl = 'https://modelscope.cn'; + const link = `${baseUrl}/studios`; + + const { data } = await got.put(`${baseUrl}/api/v1/studios`, { + json: { + PageSize: ctx.query.limit ? parseInt(ctx.query.limit) : 36, + PageNumber: 1, + SortBy: 'gmt_modified', + }, + }); + + const studios = data.Data.Studios.map((studio) => ({ + title: studio.ChineseName || studio.Name, + description: studio.Description, + author: studio.CreatedBy, + link: `${link}/${studio.Path}/${studio.Name}`, + pubDate: parseDate(studio.CreatedTime, 'X'), + category: studio.Tags, + slug: `/${studio.Path}/${studio.Name}`, + coverImage: studio.CoverImage.startsWith('https://img.alicdn.com/') ? undefined : studio.CoverImage, + })); + + const items = await Promise.all( + studios.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data } = await got(`${baseUrl}/api/v1/studio${item.slug}`); + + const content = data.Data.ReadMeContent; + item.description = art(path.join(__dirname, 'templates/desc.art'), { + coverImage: item.coverImage, + description: item.description, + md: md.render(content), + }); + + return item; + }) + ) + ); + + ctx.state.data = { + title: '创空间首页 · 魔搭社区', + description: 'ModelScope——汇聚各领域先进的机器学习模型，提供模型探索体验、推理、训练、部署和应用的一站式服务。在这里，共建模型开源社区，发现、学习、定制和分享心仪的模型。', + image: 'https://g.alicdn.com/sail-web/maas/0.8.10/favicon/128.ico', + link, + item: items, + }; +}; diff --git a/lib/v2/modelscope/templates/community.art b/lib/v2/modelscope/templates/community.art new file mode 100644 index 00000000000000..7dafedaa716f54 --- /dev/null +++ b/lib/v2/modelscope/templates/community.art @@ -0,0 +1,11 @@ +{{ if thumb }} +
+{{ /if }} + +{{ if quote }} +
{{ quote }}
+{{ /if }} + +{{ if content }} + {{@ content }} +{{ /if }} diff --git a/lib/v2/modelscope/templates/desc.art b/lib/v2/modelscope/templates/desc.art new file mode 100644 index 00000000000000..5a0d9d9501597a --- /dev/null +++ b/lib/v2/modelscope/templates/desc.art @@ -0,0 +1,9 @@ +{{ if coverImage }} +
+{{ /if }} + +{{ if description }} + {{ description }}
+{{ /if }} + +{{@ md }}