Quantcast
Channel: Active questions tagged feedparser+rss+python-2.7 - Stack Overflow
Viewing all articles
Browse latest Browse all 10

How to detect with feedparser if there are new items in an RSS channel?

$
0
0

I have the following code. When you understand the code, you can look at the two comments with the capital letters. I could test if there are new items in the channel with insert or ignore but I'm trying the better mechanism with utilization feed.updated_parsed attribute. Why doesn't it work as expected?

from __future__ import unicode_literals
import feedparser
from sqlite3  import dbapi2 as sqlite
import sys, os
from datetime import datetime
from time import mktime
from daeutils import *
import re
import random
import optparse
import curses
import socket

def getActiveChannels():
  """Returns a list of active RSS channels"""
  con = sqlite.connect(connectionString)
  cur = con.cursor()
  cur.execute("select id, title, xmlurl, updated from channels")
  channels = cur.fetchall()
  cur.close()
  con.close()
  return channels

def getItemsForChannel(xmlUrl, lastUpdate):   
  socket.setdefaulttimeout(60)
  feedparserDictionary = feedparser.parse(xmlUrl)
  updatedTime = datetime.fromtimestamp(mktime(feedparserDictionary.feed.updated_parsed))
  lst = datetime.strptime(lastUpdate, "%Y-%m-%dT%H:%M:%S.%f")
  if updatedTime < lst:
    return [] # HERE NOT BEHAVING CORRECTLY, WHEN I COMMENT THIS LINE, THERE MAY BE A FEW ITEMS

  items = feedparserDictionary.entries
  print "There are new %d items" % len(items)
  return items

def setChannelUpdateTime(xmlUrl, tm):
  con = sqlite.connect(connectionString)
  cur = con.cursor()
  cur.execute("update channels set updated = :tm where xmlurl = :xmlUrl", locals())
  con.commit()
  print "updated successfully"
  cur.close()
  con.close()

if __name__ == "_main__":
   con = sqlite.connect(connectionString)
   for channel in getActiveChannels():
     channelId, channelTitle, channelXmlUrl, lastChannelUpdate = channel
     countOfNewItems = 0
     items = getItemsForChannel(channelXmlUrl, lastChannelUpdate)

     for item in items:
       title, link, description, priority, updated = item
       cur = con.cursor()
       cur.execute("insert or ignore into feeds \
              (title, link, description, read, updated, channelid) \
              values (?, ?, ?, ?, ?, ?)", \
              (title, link, description, 0, updated, channelId))

       countOfNewItems += cur.rowcount # WHICH ARE INSERTED HERE
       con.commit()
       cur.close()


     if countOfNewItems:
       print "Found new items"
       now = datetime.now().isoformat()
       if "." not in now:
         now = now + ".000000"
       setChannelUpdateTime(channelXmlUrl, now)

Here are the two tables in sqlite:

CREATE TABLE channels (id integer primary key, title string, text string, description string, type string, xmlurl string unique, htmlurl string, priority integer, active integer, deactivated integer, updated text);
CREATE TABLE feeds (id integer primary key, title string, link string unique, description string, read integer, priority integer, updated string, channelid integer, foreign key (channelid) references channels(id));

Viewing all articles
Browse latest Browse all 10

Latest Images

Trending Articles



Latest Images

<script src="https://jsc.adskeeper.com/r/s/rssing.com.1596347.js" async> </script>