We Keep Coding

How to correctly use 'async, await and promises' in nodejs, while allocating values to a variable returned from a time-consuming function?

Problem Statement:
Our aim is to allocate values in the array ytQueryAppJs, which are returned from a time consuming function httpsYtGetFunc().
The values in ytQueryAppJs needs to be used many times in further part of the code, hence it needs to be done 'filled', before the code proceeds further.
There are many other arrays like ytQueryAppJs, namely one of them is ytCoverAppJs, that needs to be allocated the value, the same way as ytQueryAppJs.
The values in ytCoverAppJs further require the use of values from ytQueryAppJs. So a solution with clean code would be highly appreciated.
(I am an absolute beginner. I have never used async, await or promises and I'm unaware of the correct way to use it. Please guide.)
Flow (to focus on):
The user submits a queryValue in index.html.
An array ytQueryAppJs is logged in console, based on the query.
Expected Log in Console (similar to):
Current Log in Console:
Flow (originally required by the project):
User submits query in index.html.
The values of arrays, ytQueryAppJs, ytCoverAppJs, ytCoverUniqueAppJs, ytLiveAppJs, ytLiveUniqueAppJs gets logged in the console, based on the query.
Code to focus on, from 'app.js':
// https://stackoverflow.com/a/14930567/14597561
function compareAndRemove(removeFromThis, compareToThis) {
return (removeFromThis = removeFromThis.filter(val => !compareToThis.includes(val)));
}
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
let promise = new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = []
response.on('data', (d) => {
chunks.push(d)
})
// Parsing the chunks
response.on('end', () => {
resultOfYtGetFunc = JSON.parse((Buffer.concat(chunks).toString()))
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < (resultOfYtGetFunc.items).length; i++) {
extractedResultOfYtGetFunc[i] = resultOfYtGetFunc.items[i].id.videoId;
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
})
})
})
let result = await promise;
return result;
}
app.post("/", function(req, res) {
// Accessing the queryValue, user submitted in index.html. We're using body-parser package here.
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Complete app.post method from app.js:
(For better understanding of the problem.)
app.post("/", function(req, res) {
// Accessing the queryValue user submitted in index.html.
query = req.body.queryValue;
// Fetcing top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
// Fetching 'cover' songs related to user's query and putting them in the array.
if (query.includes("cover") == true) {
ytCoverAppJs = httpsYtGetFunc(query);
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
} else {
ytCoverAppJs = httpsYtGetFunc(query + " cover");
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
}
// Fetching 'live performances' related to user's query and putting them in the array.
if (query.includes("live") == true) {
ytLiveAppJs = httpsYtGetFunc(query);
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
} else {
ytLiveAppJs = httpsYtGetFunc(query + " live");
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
}
// Emptying all the arrays.
ytQueryAppJs.length = 0;
ytCoverAppJs.length = 0;
ytCoverUniqueAppJs.length = 0;
ytLiveAppJs.length = 0;
ytLiveUniqueAppJs.length = 0;
});

Unfortunately you can use the async/await on http module when making requests. You can install and use axios module . In your case it will be something like this
const axios = require('axios');
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
const promise = axios.get(urlOfYtGetFunc).then(data => {
//do your data manipulations here
})
.catch(err => {
//decide what happens on error
})
Or async await
const data = await axios.get(urlOfYtGetFunc);
//Your data variable will become what the api has returned
If you still want to catch errors on async await you can use try catch
try{
const data = await axios.get(urlOfYtGetFunc);
}catch(err){
//In case of error do something
}

I have just looked at the code I think the issue is how you are handling the async code in the request handler. You are not awaiting the result of the function call to httpsYtGetFunc in the body so when it returns before the promise is finished which is why you get the Promise {Pending}.
Another issue is that the array is not extractedResultOfYtGetFunc is not initialised and you may access indexes that don't exist. The method to add an item to the array is push.
To fix this you need to restructure your code slightly. A possible solution is something like this,
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI";
urlOfYtGetFunc =
"https://www.googleapis.com/youtube/v3/search?key=" +
apiKey +
"&part=snippet&q=" +
queryOfYtGetFunc +
"&maxResults=4&order=relevance&type=video";
return new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = [];
response.on("data", (d) => {
chunks.push(d);
});
// Parsing the chunks
response.on("end", () => {
// Initialising the array
extractedResultOfYtGetFunc = []
resultOfYtGetFunc = JSON.parse(Buffer.concat(chunks).toString());
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < resultOfYtGetFunc.items.length; i++) {
// Adding the element to the array
extractedResultOfYtGetFunc.push(resultOfYtGetFunc.items[i].id.videoId);
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
});
});
});
}
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = await httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Another option would be to use axios,
The code for this would just be,
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
try{
ytQueryAppJs = await axios.get(url); // replace with your URL
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
} catch(e) {
console.log(e);
}
});
Using Axios would be a quicker way as you don't need to write promise wrappers around everything, which is required as the node HTTP(S) libraries don't support promises out of the box.

How to use loops with fetch and Promise.all?

So I was following along an MDN article on promises and was wondering how to modify the following code to be able to work for any number of files (not just 3).
function fetchAndDecode(url) {
return fetch(url).then(response => {
if(!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
} else {
if(response.headers.get("content-type") === "image/jpeg") {
return response.blob();
} else if(response.headers.get("content-type") === "text/plain") {
return response.text();
}
}
})
.catch(e => {
console.log(`There has been a problem with your fetch operation for resource "${url}": ` + e.message);
})
.finally(() => {
console.log(`fetch attempt for "${url}" finished.`);
})
}
let coffee = fetchAndDecode('coffee.jpg');
let tea = fetchAndDecode('tea.jpg');
let description = fetchAndDecode('description.txt');
Promise.all([coffee, tea, description]).then(values => {
console.log(values);
// Store each value returned from the promises in separate variables; create object URLs from the blobs
let objectURL1 = URL.createObjectURL(values[0]);
let objectURL2 = URL.createObjectURL(values[1]);
let descText = values[2];
// Display the images in <img> elements
let image1 = document.createElement('img');
let image2 = document.createElement('img');
image1.src = objectURL1;
image2.src = objectURL2;
document.body.appendChild(image1);
document.body.appendChild(image2);
// Display the text in a paragraph
let para = document.createElement('p');
para.textContent = descText;
document.body.appendChild(para);
});
MDN specifically notes that "If you were improving this code, you might want to loop through a list of items to display, fetching and decoding each one, and then loop through the results inside Promise.all(), running a different function to display each one depending on what the type of code was. This would make it work for any number of items, not just three." I'm not sure how to do this though, and would appreciate help. Thanks.

The second part of the code could be generalised as follows:
let urls = ['coffee.jpg', 'tea.jpg', 'description.txt'];
Promise.all(urls.map(fetchAndDecode)).then(values => {
let elem;
for (let value of values) {
if (value instanceof Blob) {
elem = document.createElement('img');
elem.src = URL.createObjectURL(value);
} else if (typeof value === "string") {
elem = document.createElement('p');
elem.textContent = value;
} else {
console.log("unexpected value type");
continue;
}
document.body.appendChild(elem);
}
});

const resources = ['coffee.jpg', 'tea.jpg', 'description'];
const resourceRequests = resources.map(fetchAndDecode);
Promise.all(resourceRequests.then(values => {
...
Is one way to implement the suggestion. This approach allows for easier modification of the list of resources, but doesn't really change any of the Promise code.
The .map code above is equivalent to (resource => fetchAndDecode(resource)) since fetchAndDecode takes only the first argument that .map would pass to it.

Dealing with multiple asynchronous function calls in a for loop

I am trying to do multiple asynchronous actions: Axios requests inside of a for loop. I want to do something after everything is resolved but there is so much going on I don't know how to do it.
I thought of making my sourcer function async and awaiting it on each iteration (and wrapping the for loop in an async function), but one problem is that sourcer doesn't actually return anything. I don't know how to return from sourcer from inside an Axios "finally" clause. Another problem is that I don't want to await each sourcer call because it would be a hit on performance.
Promise.all sounds like the right direction to take but I don't know how to implement it with this for loop.
Here is the relevant part of my code (ts is a large array of objects):
.then(ts => {
// Create an association object that determines each media item's source
const sourcer = media => { // Input is either [image filename, image url] or [image filename, image url, video filename, video url]
// Test to see if the original URL works
let validURL = true
axios.get(media[1])
.then(resp => {
if (resp.status.toString()[0] !== '2') validURL = false
})
.catch(resp => {
if (resp.status.toString()[0] !== '2') validURL = false
})
.finally(() => {
let newSources = JSON.parse(JSON.stringify(this.state.sources))
let newModals = JSON.parse(JSON.stringify(this.state.modals))
if (validURL) newSources[media[0]] = media[1]
// If the original URL does not work, pull media item from server
else newSources[media[0]] = `http://serveripaddress/get_media?filename=${media[0]}`
newModals[media[0]] = false
this.setState({ sources: newSources, modals: newModals })
})
if (media.length > 2) { // If the media item is a video, do the same checks
let validVURL = true
axios.get(media[3])
.then(resp => {
if (resp.status.toString()[0] !== '2') validVURL = false
})
.catch(resp => {
if (resp.status.toString()[0] !== '2') validVURL = false
})
.finally(() => {
let newSources2 = JSON.parse(JSON.stringify(this.state.sources))
let newThumbnails = JSON.parse(JSON.stringify(this.state.thumbnails))
if (validVURL) newSources2[media[2]] = media[3]
else newSources2[media[2]] = `http://serveripaddress/get_media?filename=${media[2]}`
newThumbnails[media[0]] = media[2] // Add an association for the video and its thumbnail
this.setState({ sources: newSources2, thumbnails: newThumbnails })
})
}
}
for (let t of ts) {
if (t.media) for (let m of t.media) sourcer(m)
if (t.preview_media) sourcer(t.preview_media)
if (t.video) sourcer(t.video)
}
})
I want to do something after ts has been iterated through and all sourcer calls are completed.
I'm not fishing for someone to write my code for me but a nudge in the right direction would be greatly appreciated.

axios.get will return a Promise, so simply build up your array of Promises and use Promise.all
So, in your case, instead of executing the http call and waiting on the response, just add it to your array.
Something like this will work. I removed your code that was handling the response of each individual get request. You can merge that code (or just copy/paste) into where I put the placeholder below:
.then(ts => {
// Create an association object that determines each media item's source
const sourcer = media => { // Input is either [image filename, image url] or [image filename, image url, video filename, video url]
// Test to see if the original URL works
let validURL = true;
const promises = [];
promises.push(axios.get(media[1]));
if (media.length > 2) { // If the media item is a video, do the same checks
let validVURL = true;
promises.push(axios.get(media[3]));
}
}
for (let t of ts) {
if (t.media)
for (let m of t.media) sourcer(m)
if (t.preview_media) sourcer(t.preview_media)
if (t.video) sourcer(t.video)
}
// Execute the Promises
Promise.all(promises).then( results => {
const media1 = results[0];
const media3 = results[1];
// TODO: Run your code for media1/media3 results
})
})

How would I got about keeping a file in order when using an async function on it in javascript?

so for the discord bot that im using, I'm able to create a custom playlist, and save it as a txt file. I want to read from the text file and queue up each song in it. The problem is thats it's an async function to do it so I lose order even though i want it to stay in order, would there be any way to go about this? This is what my code looks like and I'm not sure where to start to change it to make it keep order from first song to last. The playlist filelooks like this:
thunder
believer
meant to be
country roads take me home
closer remix
mean to me
And the code looks like this. Any help would be appreciated. Thanks!
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(actualPlaylist)
});
lineReader.on('line', async function(line){
console.log('Line from file:', line);
url = line ? line.replace(/<(.+)>/i, '$1') : '';
console.log(url);
try{
var aVideo = await youtube.getVideo(url);
} catch(error) {
try{
var myVideos = await youtube.searchVideos(line, 1);
var aVideo = await youtube.getVideoByID(myVideos[0].id);
} catch(myError) {
console.error(myError);
return msg.channel.send("Couldnt find any videos by that name.");
}
}
return handleVideo(aVideo, msg, voiceChannel, false); // This is a async function we create down below next to the 'play' function!
});

One way to achieve this would be to make a queue and have it unshift the songs when it started the next one. Something like this maybe. Note this code is untested. Good Luck!
/* globals youtube msg handleVideo voiceChannel */
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream('./playlist.txt')
})
let songList = []
let called = false
async function getSong (item) {
let aVideo
try {
aVideo = await youtube.getVideo(item.url)
} catch (error) {
try {
const myVideos = await youtube.searchVideos(item.line, 1)
aVideo = await youtube.getVideoByID(myVideos[0].id)
} catch (myError) {
console.error(myError)
return msg.channel.send('Couldnt find any videos by that name.')
}
}
return handleVideo(aVideo, msg, voiceChannel, false)
}
function videoQueue () {
if (!songList.length) {
called = false
return
}
const item = songList.unshift()
getSong(item)
.then(() => {
videoQueue()
})
.catch(() => {
console.log('trying to play next song')
videoQueue()
})
}
lineReader.on('line', async function (line) {
console.log('Line from file:', line)
const url = line ? line.replace(/<(.+)>/i, '$1') : ''
songList.push({url, line}) // {url, line} = {url:url, line:line}
if (!called) videoQueue()
})

node.js resolve promise and return value

I use the Microsoft bot framework to come up with a "simple" PoC bot. I used a tutorial as a basis and extend it.
I've a couple of basic functions for differet intents (ie. greetings, goodbye, etc) and one with some more logic in it (reqstatus).
The simple ones (ie greeting.js) return the answer nicely but the more complex one doesn't (reqstatus.js). Running the main code of reqstatus.js (without the first "const getReqStatus = (entity) => {") in a standalone script works.
server.js (main) -> see call in "if (intent) {"...
const getFeelings = require('./intents/feelings.js')
const getGoodbyes = require('./intents/goodbyes.js')
const getGreetings = require('./intents/greetings.js')
const getHelp = require('./intents/help.js')
const getReqStatus = require('./intents/reqstatus.js')
...
const bot = new builder.UniversalBot(connector)
// Intents based on definitions on recast
const INTENTS = {
feelings: getFeelings,
goodbyes: getGoodbyes,
greetings: getGreetings,
help: getHelp,
reqstatus: getReqStatus,
}
// Event when Message received
bot.dialog('/', (session) => {
recastClient.textRequest(session.message.text)
.then(res => {
const intent = res.intent()
const entity = res.get('request_number')
console.log(`UserName: ${session.message.user.name}`)
console.log(`Msg: ${session.message.text}`)
console.log(`Intent: ${intent.slug}`)
if (intent) {
INTENTS[intent.slug](entity)
.then(res => session.send(res))
.catch(err => session.send(err))
}
})
.catch(() => session.send('Sorry I didn\'t get that. '))
})
...
greetings.js -> Returns the string ok
const getGreetings = () => {
const answers = ['Hi, my name is SuperBot. Nice to meet you!', ]
return Promise.resolve((answers))
}
module.exports = getGreetings
reqstatus.js -> Does not return anything
const getReqStatus = (entity) => {
var request = require('request');
var request_number = entity.toLowerCase()
var output = [];
// Processing
var lineReader = require('readline').createInterface({
input: fs.createReadStream('netreqs.csv')
});
lineReader.on('line', function (line) {
var jsonFromLine = {};
var lineSplit = line.split(';');
jsonFromLine.req = lineSplit[0];
jsonFromLine.req_count = lineSplit[1];
jsonFromLine.req_type = lineSplit[2];
//...
var req_lowever = jsonFromLine.req.toLowerCase()
if (req_lowever == request_number) {
output.push( `Your request ${jsonFromLine.req} was received`);
// simplified
}
});
// Output
lineReader.on('close', function (line) {
if (output == '') {
output.push( `I was not able to find a request like ${request_number}.`);
}
console.log(output); // list output
return Promise.resolve(output);
});
}
module.exports = getReqStatus
I also tried to put getReqStatus in a function but that also didn't work.
After a lot of trying and googling I'm still stuck and wanted to ask the experts here. Thanks a lot in advance.

I think that the problem is that your getReqStatus isn't really returning anything. In your example getGreetings function you're actually returning Promise.resolve(answers) as the return value of that function.
However, in your getReqStatus function, you just set up a listener lineReader close event:
lineReader.on('close', function (line) {
if (output == '') {
output.push( `I was not able to find a request like ${request_number}.`);
}
console.log(output); // list output
return Promise.resolve(output);
});
You're returning a Promise resolved inside the anonymous callback function you're passing to lineReader.on() as second parameter. That is not the return value from the getReqStatus function itself, so that getReqStatus is not returning anything, as expected.
The code of that function runs correctly as standalone code, as you say, just because it sets the listener properly and it does what it has to do. However, that code just doesn't return a Promise when wrapped in a function.
What you would need is to return a Promise that wraps the lineReader.on close handler, like:
function getReqStatus(){
//...code
return new Promise( function(resolve , reject ){
lineReader.on('close', function (line) {
if (output == '') {
output.push( `I was not able to find a request like ${request_number}.`);
}
console.log(output); // list output
return resolve(output);
});
});
}
I say would because I really don't know if this code will work, I don't have any kind of experience with the Microsoft Bot framework and not used at all with the readline module. However, even if this doesn't solve your problem, I hope it will help you a bit understanding why your function doesn't return a Promise and how could you fix it.