Mathspace Content Crawler

node v6.17.1
version: 1.1.0
endpointsharetweet
const HOST = // Staging Data 'https://staging.mathspace.co'; // Production // 'https://mathspace.co'; const QUERY = ` query ($after: String){ admin { allProblemTemplates (after: $after) { pageInfo{ endCursor } edges { node { pk title instruction init mathyonSSetup updatedAt updatedBy { username } attachment status subproblemTemplates { edges { node { instruction mathyonRSetup stepTemplates { edges { node { expression hint secondHint } } } } } } } } } } } `; const LOGIN_ENDPOINT = '/accounts/api/login/'; const GRAPHQL_ADMIN_ENDPOINT = '/graphql/admin/';
const fetch = require('node-fetch'); const R = require('ramda'); /** * Extract a cookie from a response object. * * @param {string} host The name of the cookie you want to collect * @returns {string} The value of the cookie, or undefined */ function getCookie(name/*: string*/, response/*: response*/)/*: ?string*/ { const regex = new RegExp(`${name}\\s*=\\s*([^;]*)`); const cookie = R.compose( setCookieString => { const [, result] = setCookieString.match(regex) || []; return result; }, arr => arr.join(';'), R.path(['headers', '_headers', 'set-cookie']) )(response); return cookie; } /** * Calls the Django REST API login endpoint. * * @param {string} username The username (or email) of the account to attempt login for. * @param {string} password The password of the account to verify login. * @returns {Promise} The request's response. */ function login(username/*: string*/, password/*: string*/)/*: Promise<any>*/ { return fetch(HOST + LOGIN_ENDPOINT, { method: 'POST', headers: { 'Content-Type': 'application/json', }, credentials: 'include', body: JSON.stringify({ username, password, }), }); }
/** * Fetch a paginated list of problem template content. * * @param {string} cursor The cursor position to start this paginated result from. * @returns {Object} The JSON response from our GraphQL endpoint. */ const getContent = (query) => async function (cursor/*: string*/)/*: Promise<Object>*/ { const [csrfToken, sessionToken] = await login( process.env.MATHSPACE_USERNAME, process.env.MATHSPACE_PASSWORD ) .then(res => [getCookie('csrftoken', res), getCookie('sessionid', res)]); return await fetch(HOST + GRAPHQL_ADMIN_ENDPOINT, { method: 'POST', headers: { 'cookie': `csrftoken=${csrfToken}; sessionid=${sessionToken}`, 'Content-Type': 'application/json', 'X-CSRFToken': csrfToken, 'Referer': HOST, }, // credentials: 'include', body: JSON.stringify({ query, variables: { after: cursor, }, }), }) .then(res => res.json()); } module.exports.getContent = getContent; module.exports.problemTemplateQuery = QUERY;
Loading…

no comments

    sign in to comment