I am migrating a piece of code from mysql to mssql package of nodejs, in which its required to insert multiple rows.
Here's the sample code I am using for testing:
const db = require('mssql');
let config = {
user: 'salim',
password: 'admin',
server: 'LAPTOP-JK45R', // You can use 'localhost\\instance' to connect to named instance
database: 'master',
}
var pool;
//initialize connection pool
var connectPool = initializeConnectionPool();
function initializeConnectionPool() {
pool = new db.ConnectionPool(config);
pool.on('error', (err) => {
logger.error(err);
});
return pool.connect();;
}
connectPool.then(async () => {
let connection = await pool.request();
console.log('Got pool connection...');
var q = "INSERT INTO Person (name, address) VALUES ?";
var values = [
['John', 'Highway 71'],
['Peter', 'Lowstreet 4'],
['Amy', 'Apple st 652'],
['Hannah', 'Mountain 21']
];
let result = await connection.query(q,[values]);
console.log(`Result: ${JSON.stringify(result)}`);
});
Its giving me error:
RequestError: Incorrect syntax near '?'.
I couldn't find any thing on official npm page of mssql, so I have been trying this: Insert multiple columns and rows into SQL Server with node js
In my code I am just using pool.
I also couldn't find how to log queries using this package, so couldn't figure out what the query is being formed.
It would be great to know any of the solution.
The ? is a way to pass parameter in mysql
In mssql it seem to be like ${entries}
refer to How to pass parameter to mssql query in node js
Related
I am using Node Express API to run SQL queries to populate a dashboard of data. I am using the mssql-node package to do so. Sometimes it runs flawlessly, other times I get the following error:
[Error: [Microsoft][SQL Server Native Client 11.0]Query timeout expired]
I am creating a poolPromise with a connectionPool to the db, then I pass that object to my other controllers which run the specific queries to populate data. I run the server which initiates the db.js script and connects to MSSQL with a pool connection.
db.js:
// for connecting to sql server
const sql = require('mssql/msnodesqlv8');
// db config to connect via windows auth
const dbConfig = {
driver: 'msnodesqlv8',
connectionString: 'Driver={SQL Server Native Client 11.0};Server={my_server};Database={my_db};Trusted_Connection={yes};',
pool: {
idleTimeoutMillis: 60000
}
};
// create a connectionpool object to pass to controllers
// this should keep a sql connection open indefinitely that we can query when the server is running
const poolPromise = new sql.ConnectionPool(dbConfig)
.connect()
.then(pool => {
console.log('Connected to MSSQL');
return pool;
})
.catch(err => console.log('Database Connection Failed! Bad Config: ', err))
module.exports = { sql, poolPromise };
An example of one of my controllers and how I use the poolPromise object is below. I currently have about 7 of these controllers that run their own specific query to populate a specific element on the dashboard. The performance of the queries each run within 1-10 seconds (depending on current server load, as I am querying an enterprise production server/db, this can vary). As I mentioned earlier, the queries run flawlessly sometimes and I have no issues, but at other times I do have issues. Is this a symptom of me querying from a shared production server? Is it preferred to query from a server that has less load? Or am I doing something in my code that could be improved?
const { sql, poolPromise } = require('../db');
// function to get data
const getData = async (req, res) => {
try {
// create query parameters from user request
let id= req.query.id;
// create query from connectionPool
let pool = await poolPromise;
let qry = `
select * from tbl where id = #Id
`
let data = await pool.request()
.input('Id', sql.VarChar(sql.MAX), id)
.query(qry);
// send 200 status and return records
res.status(200);
res.send(data.recordset);
} catch(err) {
console.log('Error:');
console.log(err);
res.sendStatus(500);
}
};
module.exports = { getData };
Im new to nodejs and Im currently doing an sql to mongodb migration. I have created a script to load data to mongodb from sql queries. I created the script with the sample code from Google and it is working. But im facing below issue and need a workaround for this.
I have an sql query array and I don't need to run those queries if any of the queries has any syntax issues or any errors in the query result. (Say if the second query has syntax issue then no need to load the data of first query to mongo, currently its loading in my case). Basically if any of the query has any issue then no need to load the result in the mongo collection. And also if any issues from the mongo side no need to commit the transactions.
I have used the mongo transactions here to roll back the data if any errors. please find the below code and any help would be much appreciated.The sql and mongo credentials are mock data only.
config file code
var mongoCollection = 'collectionName';
exports.mongoCollection = mongoCollection;
var queryList = [
'sample query one',
'sample query two '
];
exports.queryList = queryList;
main script code
var MongoClient = require('mongodb').MongoClient;
var sql = require('mysql');
const config = require('./assets/config');
var sqlConfig = {
user: 'username',
password: 'password',
server: 'servername',
database: 'databasename',
port: 'portname',
multipleStatements: true
};
async function transaction() {
const mongodbUrl = 'mongourl';
const client = await MongoClient.connect(mongodbUrl, {useNewUrlParser: true}, {useUnifiedTopology:
true});
const db = client.db();
config.queryList.forEach(query => {
new sql.ConnectionPool(sqlConfig).connect().then(pool => {
return pool.request().query(query)
}).then(result => {
(async()=>{
const session = client.startSession();
session.startSession({
readConcers: {level: 'snapshot'},
writeConcern: {w: 'majority'}
});
try {
const collection = client.db('mongodbName').collection(config.mongoCollection);
await collection.insertMany(result.recordset, {session});
await session.commitTransaction();
session.emdSession();
console.log('transaction completed');
}catch(error){
await session.abortTransaction();
session.endSession();
console.log('transaction aborted');
throw error;
}
});
sql.close();
}).catch(error => {
sql.close();
throw error;
})
});
};
transaction();
Depending on the volume of data, you might look at breaking the process into two parts
Get the data from mySql
If no errors, load into Mongo
That would save you having to roll back the mongo writes
You can also take advantage of the default mongo pool size (5) and use pool on the mySQL side too.
Currently, this code is creating a pool for every select, which isn't optimal
config.queryList.forEach(query => {
new sql.ConnectionPool(sqlConfig).connect().then(pool => {//<-New pool per query?
return pool.request().query(query)
})
})
Instead, you can set up a pool once, per the mySql documentation
It looks like that driver only has a callback api, but you can promisfy the query to make it easier to work with.
So to put it all together, you could try something like this (this isn't working/tested code, just a suggestion)
var MongoClient = require('mongodb').MongoClient;
var sql = require('mysql');
const config = require('./assets/config');
var pool = sql.createPool({
connectionLimit : 5,
host : 'servername',
user : 'username',
password : 'password',
database : 'databasename'
});
async function transaction() {
try{
const mongodbUrl = 'mongourl';
const client = await MongoClient.connect(mongodbUrl, {useNewUrlParser: true}, {useUnifiedTopology: true});
const db = client.db();
const collection = client.db('mongodbName').collection(config.mongoCollection);
//Map your query list to an array of runSql promises
//this will complete when all queries return, and jump to the catch if any fail
let results = await Promise.all(config.queryList.map(runSql))
//Map the results to an array of mongo inserts
let inserts = await Promise.all(results.map(r=>collection.insertMany(r.recordset)))
//Close all connections
pool.end((err)=>err?console.err(err):console.log('MySQL Closed'))
client.close((err)=>err?console.err(err):console.log('MongoDB Closed'))
}
catch(err){
console.error(err)
}
};
transaction();
function runSql(queryStr){
return new Promise((resolve, reject)=>{
pool.query(queryStr, function (error, results, fields){
error?reject(error):resolve(results)
})
})
}
If data volume is a concern, you might want to look at getting streams from your mySql selects instead of just running them
I don't understand why my mongo db isn't showing up when I run "show databases" in the command line. I see other mongo db's I created in the past, but not the current one. Here is my code: (using mongoose ORM):
var mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/fuelTracker');
var Schema = mongoose.Schema;
var fuelSchema = new Schema({
time : { type : Date, default: Date.now },
miles : Number,
gallons: Number
});
var FuelStop = mongoose.model('FuelStop', fuelSchema);
module.exports = FuelStop;
And where I'm attempting a basic model.save operation:
app.post('/', function (req, res ) {
results = req.body;
var fuelStop = new FuelStop (results)
fuelStop.save(function() {
console.log('record saved to monogoDB');
});
})
Any clue as to why my 'fuelTracker' database doesn't appear in the command line when I run 'show databases' within mongo?
THANK YOU!!
did you try show dbs? show databases print all databases available, see the doc here: https://docs.mongodb.com/manual/reference/mongo-shell/
The database doesn't get created until you insert data into a collection in the database.
I tested your code by creating and running a script to seed the fuelTracker database with example JSON data. I then ran show databases and was able to see fuelTracker listed.
If you would like to try this, in a new file seed.js:
const db = require('./fileName.js');
const fs = require('fs');
let fuelData = fs.readFileSync('./data.json', 'utf8', (err, data) => {
if (err) throw err;
console.log(data);
});
let jsonFuel = JSON.parse(fuelData);
db.remove({}, () => {
console.log('Successfully removed fuel data');
db.collection.insert(jsonFuel, (err, docs) => {
if (err) {
console.log(`error inserting data: ${err}`);
} else {
console.log(`Fuel data was stored: ${docs}`);
}
});
});
Then create example data in data.json:
[
{
"time":"04/18/2018",
"miles":"28",
"gallons":"30"
}
]
Then run your seed file and your database should hopefully be showing up.
Question Updated:
The solution should detail a simplified, proper model of a Node connection module allowing for the re-use of the connection by any module of a Node application needing to connect to a database. That way, the question might be useful for anyone with connection module issues in Node.
The answer might even include a way to pass in different credentials in order to connect to different tables or databases with a single function call from anywhere in an app.
I wrote a script that utilized a number of simple modules to allow a user to post login data, have that data validated on the server, and if correct, receive a response of success. A very basic login functionality.
The problem: One user can log-in, but any more log-in attempts before I restart the server fail to connect to the database.
It appears that, because I'm declaring the connection in a variable in my module db_connect and requiring that module, the connection can't be re-used. It's being declared in the required module, and I had mistakenly believed that calling the variable in each connection attempt would recreate the connection. It doesn't.
The solution: As suggested by barry in the comments, in the db_connect module, I need to make the connection functionality a function rather than a variable, so I can then create the connection from within my validation script.
How can I do this? I'm trying to output the connection object when calling the createConnection() function, which is an exported method of db_connect.
db_connect:
console.log('db_connect module initialized');
var mysql = require('mysql');
function createConnection(){
var connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : '',
database : 'officeball'
});
}
exports.createConnection = createConnection();
exports.mysql = mysql;
validator:
console.log('validator module initialized');
var connect = require("./db_connect");
function validate(username, password, callback){
var createConnection = connect.createConnection();
//the idea is for this to return the object, connection,
//which opens a new connection
connection.connect(function (err){
if (err) return callback(new Error('Failed to connect'), null);
console.log('Connection with the Officeball MySQL database openned...');
connection.query('select username,password,fname,lname,rank,active from users where username=?',
username,
function(err,rows,fields) {
connection.destroy();
console.log('...Connection with the Officeball MySQL database closed.');
if (err)
return callback(new Error ('Error while performing query'), null);
if (rows.length !== 1)
return callback(new Error ('- [Anomaly] - Failed to find exactly one user'), null);
if (rows[0].password === password & rows[0].active === "yes") {
var result = new Object;
result.username = rows[0].username;
result.password = rows[0].password;
result.fname = rows[0].fname;
result.lname = rows[0].lname;
result.rank = rows[0].rank;
return callback(null, result);
} if(rows[0].active !== "yes"){
return callback(new Error ('User account not active.'), null);
}else {
return callback(new Error ('Login credentials did not match.'), null);
}
});
});
};
exports.validate = validate;
Console log (originally, there was a connection error, but after my fix-attempt, the error is about the method):
C:\xampp\htdocs\officeball\node_scripts>node index.js
application initialized
server module initialized
login module initialized
validator module initialized
db_connect module initialized
sale module initialized
Server running at http://127.0.0.1:8080/
User username1 is attempting login...
TypeError: Property 'createConnection' of object #<Object> is not a function
at Object.validate (C:\xampp\htdocs\officeball\node_scripts\custom_modules\v
alidator.js:6:33)
at C:\xampp\htdocs\officeball\node_scripts\custom_modules\login.js:61:13
at callbacks (C:\xampp\htdocs\officeball\node_scripts\node_modules\express\l
ib\router\index.js:164:37)
at param (C:\xampp\htdocs\officeball\node_scripts\node_modules\express\lib\r
outer\index.js:138:11)
at pass (C:\xampp\htdocs\officeball\node_scripts\node_modules\express\lib\ro
uter\index.js:145:5)
at Router._dispatch (C:\xampp\htdocs\officeball\node_scripts\node_modules\ex
press\lib\router\index.js:173:5)
at Object.router (C:\xampp\htdocs\officeball\node_scripts\node_modules\expre
ss\lib\router\index.js:33:10)
at next (C:\xampp\htdocs\officeball\node_scripts\node_modules\express\node_m
odules\connect\lib\proto.js:193:15)
at multipart (C:\xampp\htdocs\officeball\node_scripts\node_modules\express\n
ode_modules\connect\lib\middleware\multipart.js:93:27)
at C:\xampp\htdocs\officeball\node_scripts\node_modules\express\node_modules
\connect\lib\middleware\bodyParser.js:64:9
Your problem was primarily what I mentioned in my comment, so I will make it an answer. Don't hurt yourself slapping your forehead. :-) See my inline comments below. You had one method invocation too many, an unreturned result, and a misnamed variable - otherwise, it works just fine.
db_connect.js
console.log('db_connect module initialized');
var mysql = require('mysql');
function createConnection(){
var connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : '',
database : 'officeball'
});
// ************ NOTE BELOW FOR CHANGE **************
// You didn't return anything from this function. You need to return the connection
return connection;
}
// ************ NOTE BELOW FOR CHANGE **************
// You are exporting a single connection by invoking createConnection();
// exports.createConnection = createConnection();
// what you want is:
exports.createConnection = createConnection;
exports.mysql = mysql;
validator.js
function validate(username, password, callback){
// ************ NOTE BELOW FOR CHANGE **************
// You had:
// var createConnection = connect.createConnection();
// but based on your code, you wanted to write this instead:
var connection = connect.createConnection();
/// ... REMAINDER OMITTED, because it was A-OK and this is already a long page
};
If you make those two three changes, you should be good to go. As ever, feel free to ask for any clarifications if it would be helpful.
Update This is how I am calling it - as you can see, I had it try to do it ever 2 seconds.
jt-test.js
var v = require('./validator');
var timers = require('timers');
var connections = 0;
timers.setInterval(function(){
v.validate('bagehot','foo',function(err,result){
if (err)
console.log('failed', err);
else
console.log('success! ',result);
});
},2000);
result
Connection with the Officeball MySQL database openned...
...Connection with the Officeball MySQL database closed.
success! { username: 'bagehot',
password: 'foo',
fname: 'walter',
lname: 'bagehot',
rank: 12 }
Connection with the Officeball MySQL database openned...
...Connection with the Officeball MySQL database closed.
success! { username: 'bagehot',
password: 'foo',
fname: 'walter',
lname: 'bagehot',
rank: 12 }
Connection with the Officeball MySQL database openned...
...Connection with the Officeball MySQL database closed.
success! { username: 'bagehot',
password: 'foo',
fname: 'walter',
lname: 'bagehot',
rank: 12 }
Connection with the Officeball MySQL database openned...
...Connection with the Officeball MySQL database closed.
success! { username: 'bagehot',
password: 'foo',
fname: 'walter',
lname: 'bagehot',
rank: 12 }
And so on. It runs indefinitely. Complete code for the three files is in this gist
You can checkout my emysql package.
https://www.npmjs.org/package/emysql
https://github.com/yanke-guo/enhanced-mysql-pool
Maybe it's some kind of outdated with the most recent node-mysql package, but still functional.
I'm getting really confused over how to connect to MongoLab on Heroku.
To connect using the uri to Heroku, I was trying to follow this example:
http://experiencecraftsmanship.wordpress.com/2012/01/06/heroku-node-js-mongodb-featuring-the-native-driver/
I looked at both his web.js and deep.js.
They both do something like:
connect.createServer(
require( 'connect-jsonrpc' )( contacts )
).listen( port );
But then only the database query in 'contacts' get passed into this server then?
Am I allowed to do multiple connect.createServer for each of my database access method?
The following is part of my code when just connecting to MongoDB locally. I am unsure of how to modify it to connect to MongoLab on Heroku.
Can someone teach me how to modify my code to connect? Or explain some of these concepts? I have no idea why the author of that website I posted used so many callbacks to do a database call, when my approach below seems straightforward enough (I'm new to JavaScript, not good with callbacks).
var app = module.exports = express.createServer(
form({ keepExtensions: true })
);
var Db = require('mongodb').Db;
var Server = require('mongodb').Server;
var client = new Db('blog', new Server('127.0.0.1', 27017, {}));
var posts;
var getAllPosts = function(err, collection) {
collection.find().toArray(function(err, results) {
posts = results;
console.log(results);
client.close();
});
}
app.get('/', function(req, response) {
client.open(function(err, pClient) {
client.collection('posts', getAllPosts);
});
// some code
response.render('layout', { posts: posts, title: 'Raymond', contentPage: 'blog' });
});
You connect to your mongolab database (so you can't create a new "blog" database). process.env.MONGOLAB_URI includes the database name as well. See your mongolab uri:
heroku config | grep MONGOLAB_URI
It looks like: mongodb://heroku_app123456:password#dbh73.mongolab.com:27737/heroku_app123456
On github there is an example how to connect and retrieve data from a mongolab database.
Use "connect" to connect to mongo, instead of defining db, server, client:
var connect = require('connect');
var mongo = require('mongodb');
var database = null;
var mongostr = [YOUR MONGOLAB_URI];
mongo.connect(mongostr, {}, function(error, db)
{
console.log("connected, db: " + db);
database = db;
database.addListener("error", function(error){
console.log("Error connecting to MongoLab");
});
});