[MONGOOSE] periodic fetch and store using mongoose & axios - fourslickz/notes GitHub Wiki

1. project setup

npm init -y
npm install mongoose axios

2. Set Up MongoDB Connection

const mongoose = require('mongoose');

// MongoDB connection URI
const uri = 'mongodb://localhost:27017/your-db-name';

mongoose.connect(uri, { useNewUrlParser: true, useUnifiedTopology: true })
  .then(() => console.log('Connected to MongoDB'))
  .catch((error) => console.error('Error connecting to MongoDB:', error));

// Define a schema and model
const DataSchema = new mongoose.Schema({
  timestamp: { type: Date, default: Date.now },
  data: mongoose.Schema.Types.Mixed,  // To store any kind of data (string, object, etc.)
});

const DataModel = mongoose.model('Data', DataSchema);

3. Fetch Data Periodically

const axios = require('axios');
const mongoose = require('mongoose');
const DataModel = require('./models/DataModel'); // Assuming you save your model separately

// Function to fetch data from an external API or any source
async function fetchData() {
  try {
    // Example: fetching data from a public API
    const response = await axios.get('https://api.example.com/data');
    const fetchedData = response.data;

    // Save the fetched data to MongoDB
    const data = new DataModel({ data: fetchedData });
    await data.save();

    console.log('Data saved at', new Date());
  } catch (error) {
    console.error('Error fetching data:', error);
  }
}

// Run the function periodically (e.g., every 10 minutes)
const intervalTime = 10 * 60 * 1000; // 10 minutes in milliseconds

setInterval(() => {
  console.log('Fetching data...');
  fetchData();
}, intervalTime);

// Run the task immediately when the app starts
fetchData();

4. Run the App

node app.js

5. Optional: Use node-cron for Scheduling

npm install node-cron

const cron = require('node-cron');
const axios = require('axios');
const mongoose = require('mongoose');
const DataModel = require('./models/DataModel'); // Assuming your Mongoose model

// Function to fetch data
async function fetchData() {
  try {
    const response = await axios.get('https://api.example.com/data');
    const fetchedData = response.data;

    const data = new DataModel({ data: fetchedData });
    await data.save();

    console.log('Data saved at', new Date());
  } catch (error) {
    console.error('Error fetching data:', error);
  }
}

// Schedule a cron job to run every 10 minutes
cron.schedule('*/10 * * * *', () => {
  console.log('Running scheduled task...');
  fetchData();
});

// Fetch data immediately on startup
fetchData();