Mathspace Content Crawler
const HOST =
// Staging Data
'https://staging.mathspace.co';
// Production
// 'https://mathspace.co';
const QUERY = `
query ($after: String){
admin {
allProblemTemplates (after: $after) {
pageInfo{
endCursor
}
edges {
node {
pk
title
instruction
init
mathyonSSetup
updatedAt
updatedBy {
username
}
attachment
status
subproblemTemplates {
edges {
node {
instruction
mathyonRSetup
stepTemplates {
edges {
node {
expression
hint
secondHint
}
}
}
}
}
}
}
}
}
}
}
`;
const LOGIN_ENDPOINT = '/accounts/api/login/';
const GRAPHQL_ADMIN_ENDPOINT = '/graphql/admin/';
const fetch = require('node-fetch');
const R = require('ramda');
/**
* Extract a cookie from a response object.
*
* @param {string} host The name of the cookie you want to collect
* @returns {string} The value of the cookie, or undefined
*/
function getCookie(name/*: string*/, response/*: response*/)/*: ?string*/ {
const regex = new RegExp(`${name}\\s*=\\s*([^;]*)`);
const cookie = R.compose(
setCookieString => {
const [, result] = setCookieString.match(regex) || [];
return result;
},
arr => arr.join(';'),
R.path(['headers', '_headers', 'set-cookie'])
)(response);
return cookie;
}
/**
* Calls the Django REST API login endpoint.
*
* @param {string} username The username (or email) of the account to attempt login for.
* @param {string} password The password of the account to verify login.
* @returns {Promise} The request's response.
*/
function login(username/*: string*/, password/*: string*/)/*: Promise<any>*/ {
return fetch(HOST + LOGIN_ENDPOINT, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
body: JSON.stringify({
username,
password,
}),
});
}
/**
* Fetch a paginated list of problem template content.
*
* @param {string} cursor The cursor position to start this paginated result from.
* @returns {Object} The JSON response from our GraphQL endpoint.
*/
const getContent = (query) => async function (cursor/*: string*/)/*: Promise<Object>*/ {
const [csrfToken, sessionToken] = await login(
process.env.MATHSPACE_USERNAME,
process.env.MATHSPACE_PASSWORD
)
.then(res => [getCookie('csrftoken', res), getCookie('sessionid', res)]);
return await fetch(HOST + GRAPHQL_ADMIN_ENDPOINT, {
method: 'POST',
headers: {
'cookie': `csrftoken=${csrfToken}; sessionid=${sessionToken}`,
'Content-Type': 'application/json',
'X-CSRFToken': csrfToken,
'Referer': HOST,
},
// credentials: 'include',
body: JSON.stringify({
query,
variables: {
after: cursor,
},
}),
})
.then(res => res.json());
}
module.exports.getContent = getContent;
module.exports.problemTemplateQuery = QUERY;
no comments