MythTV: datadirect.cpp Source File

Go to the documentation of this file.
00001 #include <unistd.h>
00002 #include <zlib.h>
00003 
00004 // Qt headers
00005 #include <QDir>
00006 #include <QFileInfo>
00007 #include <QByteArray>
00008 #include <QNetworkReply>
00009 #include <QAuthenticator>
00010 
00011 // MythTV headers
00012 #include "datadirect.h"
00013 #include "sourceutil.h"
00014 #include "channelutil.h"
00015 #include "frequencytables.h"
00016 #include "listingsources.h"
00017 #include "mythcontext.h"
00018 #include "mythdb.h"
00019 #include "mythlogging.h"
00020 #include "mythversion.h"
00021 #include "mythmiscutil.h"
00022 #include "dbutil.h"
00023 #include "mythsystem.h"
00024 #include "exitcodes.h"
00025 #include "mythdownloadmanager.h"
00026 #include "mythtvexp.h"
00027 
00028 #define LOC QString("DataDirect: ")
00029 
00030 static QMutex  user_agent_lock;
00031 static QString user_agent;
00032 
00033 static QMutex lineup_type_lock;
00034 static QMap<QString,uint> lineupid_to_srcid;
00035 static QMap<uint,QString> srcid_to_type;
00036 
00037 static void    set_lineup_type(const QString &lineupid, const QString &type);
00038 static QString get_lineup_type(uint sourceid);
00039 static QString get_setting(QString line, QString key);
00040 static bool    has_setting(QString line, QString key);
00041 static QString html_escape(QString str);
00042 static void    get_atsc_stuff(QString channum, int sourceid, int freqid,
00043                               int &major, int &minor, long long &freq);
00044 static QString process_dd_station(uint sourceid,
00045                                   QString  chan_major, QString  chan_minor,
00046                                   QString &tvformat,   uint    &freqid);
00047 static uint    update_channel_basic(uint    sourceid,   bool    insert,
00048                                     QString xmltvid,    QString callsign,
00049                                     QString name,       uint    freqid,
00050                                     QString chan_major, QString chan_minor);
00051 void authenticationCallback(QNetworkReply *reply, QAuthenticator *auth,
00052                             void *arg);
00053 QByteArray gUncompress(const QByteArray &data);
00054 
00055 DataDirectStation::DataDirectStation(void) :
00056     stationid(""),              callsign(""),
00057     stationname(""),            affiliate(""),
00058     fccchannelnumber("")
00059 {
00060 }
00061 
00062 DataDirectLineup::DataDirectLineup() :
00063     lineupid(""), name(""), displayname(""), type(""), postal(""), device("")
00064 {
00065 }
00066 
00067 DataDirectLineupMap::DataDirectLineupMap() :
00068     lineupid(""), stationid(""), channel(""), channelMinor("")
00069 {
00070 }
00071 
00072 DataDirectSchedule::DataDirectSchedule() :
00073     programid(""),              stationid(""),
00074     time(QDateTime()),          duration(QTime()),
00075     repeat(false),              isnew(false),
00076     stereo(false),              dolby(false),
00077     subtitled(false),
00078     hdtv(false),                closecaptioned(false),
00079     tvrating(""),
00080     partnumber(0),              parttotal(0)
00081 {
00082 }
00083 
00084 DataDirectProgram::DataDirectProgram() :
00085     programid(""),  seriesid(""),      title(""),
00086     subtitle(""),   description(""),   mpaaRating(""),
00087     starRating(""), duration(QTime()), year(""),
00088     showtype(""),   colorcode(""),     originalAirDate(QDate()),
00089     syndicatedEpisodeNumber("")
00090 {
00091 }
00092 
00093 DataDirectProductionCrew::DataDirectProductionCrew() :
00094     programid(""), role(""), givenname(""), surname(""), fullname("")
00095 {
00096 }
00097 
00098 DataDirectGenre::DataDirectGenre() :
00099     programid(""), gclass(""), relevance("")
00100 {
00101 }
00102 
00103 // XXX Program duration should be stored as seconds, not as a QTime.
00104 //     limited to 24 hours this way.
00105 
00106 bool DDStructureParser::startElement(const QString &pnamespaceuri,
00107                                      const QString &plocalname,
00108                                      const QString &pqname,
00109                                      const QXmlAttributes &pxmlatts)
00110 {
00111     (void)pnamespaceuri;
00112     (void)plocalname;
00113 
00114     currtagname = pqname;
00115     if (currtagname == "xtvd")
00116     {
00117         QString   beg   = pxmlatts.value("from");
00118         QDateTime begts = QDateTime::fromString(beg, Qt::ISODate);
00119         parent.SetDDProgramsStartAt(begts);
00120 
00121         QString   end   = pxmlatts.value("to");
00122         QDateTime endts = QDateTime::fromString(end, Qt::ISODate);
00123         parent.SetDDProgramsEndAt(endts);
00124     }
00125     else if (currtagname == "station")
00126     {
00127         curr_station.Reset();
00128         curr_station.stationid = pxmlatts.value("id");
00129     }
00130     else if (currtagname == "lineup")
00131     {
00132         curr_lineup.Reset();
00133         curr_lineup.name = pxmlatts.value("name");
00134         curr_lineup.type = pxmlatts.value("type");
00135         curr_lineup.device = pxmlatts.value("device");
00136         curr_lineup.postal = pxmlatts.value("postalCode");
00137         curr_lineup.lineupid = pxmlatts.value("id");
00138         curr_lineup.displayname = curr_lineup.name + "-" + curr_lineup.type +
00139             "-" + curr_lineup.device + "-" +
00140             curr_lineup.postal + "-" +
00141             curr_lineup.lineupid;
00142 
00143         if (curr_lineup.lineupid.isEmpty())
00144         {
00145             curr_lineup.lineupid = curr_lineup.name + curr_lineup.postal +
00146                 curr_lineup.device + curr_lineup.type;
00147         }
00148     }
00149     else if (currtagname == "map")
00150     {
00151         int tmpindex;
00152         curr_lineupmap.Reset();
00153         curr_lineupmap.lineupid = curr_lineup.lineupid;
00154         curr_lineupmap.stationid = pxmlatts.value("station");
00155         curr_lineupmap.channel = pxmlatts.value("channel");
00156         tmpindex = pxmlatts.index("channelMinor"); // for ATSC
00157         if (tmpindex != -1)
00158             curr_lineupmap.channelMinor = pxmlatts.value(tmpindex);
00159     }
00160     else if (currtagname == "schedule")
00161     {
00162         curr_schedule.Reset();
00163         curr_schedule.programid = pxmlatts.value("program");
00164         curr_schedule.stationid = pxmlatts.value("station");
00165 
00166         QString timestr = pxmlatts.value("time");
00167         QDateTime UTCdt = QDateTime::fromString(timestr, Qt::ISODate);
00168 
00169         curr_schedule.time = MythUTCToLocal(UTCdt);
00170         QString durstr;
00171 
00172         durstr = pxmlatts.value("duration");
00173         curr_schedule.duration = QTime(durstr.mid(2, 2).toInt(),
00174                                        durstr.mid(5, 2).toInt(), 0, 0);
00175 
00176         curr_schedule.repeat = (pxmlatts.value("repeat") == "true");
00177         curr_schedule.isnew = (pxmlatts.value("new") == "true");
00178         curr_schedule.stereo = (pxmlatts.value("stereo") == "true");
00179         curr_schedule.dolby = (pxmlatts.value("dolby") == "Dolby" ||
00180                                pxmlatts.value("dolby") == "Dolby Digital");
00181         curr_schedule.subtitled = (pxmlatts.value("subtitled") == "true");
00182         curr_schedule.hdtv = (pxmlatts.value("hdtv") == "true");
00183         curr_schedule.closecaptioned = (pxmlatts.value("closeCaptioned") ==
00184                                         "true");
00185         curr_schedule.tvrating = pxmlatts.value("tvRating");
00186     }
00187     else if (currtagname == "part")
00188     {
00189         curr_schedule.partnumber = pxmlatts.value("number").toInt();
00190         curr_schedule.parttotal = pxmlatts.value("total").toInt();
00191     }
00192     else if (currtagname == "program")
00193     {
00194         curr_program.Reset();
00195         curr_program.programid = pxmlatts.value("id");
00196     }
00197     else if (currtagname == "crew")
00198     {
00199         curr_program.Reset();
00200         lastprogramid = pxmlatts.value("program");
00201     }
00202     else if (currtagname == "programGenre")
00203     {
00204         curr_genre.Reset();
00205         lastprogramid = pxmlatts.value("program");
00206     }
00207 
00208     return true;
00209 }
00210 
00211 bool DDStructureParser::endElement(const QString &pnamespaceuri,
00212                                    const QString &plocalname,
00213                                    const QString &pqname)
00214 {
00215     (void)pnamespaceuri;
00216     (void)plocalname;
00217 
00218     MSqlQuery query(MSqlQuery::DDCon());
00219 
00220     if (pqname == "station")
00221     {
00222         parent.m_stations[curr_station.stationid] = curr_station;
00223 
00224         query.prepare(
00225             "INSERT INTO dd_station "
00226             "     ( stationid,  callsign,  stationname, "
00227             "       affiliate,  fccchannelnumber)       "
00228             "VALUES "
00229             "     (:STATIONID, :CALLSIGN, :STATIONNAME, "
00230             "      :AFFILIATE, :FCCCHANNUM)");
00231 
00232         query.bindValue(":STATIONID",   curr_station.stationid);
00233         query.bindValue(":CALLSIGN",    curr_station.callsign);
00234         query.bindValue(":STATIONNAME", curr_station.stationname);
00235         query.bindValue(":AFFILIATE",   curr_station.affiliate);
00236         query.bindValue(":FCCCHANNUM",  curr_station.fccchannelnumber);
00237 
00238         if (!query.exec())
00239             MythDB::DBError("Inserting into dd_station", query);
00240     }
00241     else if (pqname == "lineup")
00242     {
00243         set_lineup_type(curr_lineup.lineupid, curr_lineup.type);
00244 
00245         parent.m_lineups.push_back(curr_lineup);
00246 
00247         query.prepare(
00248             "INSERT INTO dd_lineup "
00249             "     ( lineupid,  name,  type,  device,  postal) "
00250             "VALUES "
00251             "     (:LINEUPID, :NAME, :TYPE, :DEVICE, :POSTAL)");
00252 
00253         query.bindValue(":LINEUPID",    curr_lineup.lineupid);
00254         query.bindValue(":NAME",        curr_lineup.name);
00255         query.bindValue(":TYPE",        curr_lineup.type);
00256         query.bindValue(":DEVICE",      curr_lineup.device);
00257         query.bindValue(":POSTAL",      curr_lineup.postal);
00258 
00259         if (!query.exec())
00260             MythDB::DBError("Inserting into dd_lineup", query);
00261     }
00262     else if (pqname == "map")
00263     {
00264         parent.m_lineupmaps[curr_lineupmap.lineupid].push_back(curr_lineupmap);
00265 
00266         query.prepare(
00267             "INSERT INTO dd_lineupmap "
00268             "     ( lineupid,  stationid,  channel,  channelMinor) "
00269             "VALUES "
00270             "     (:LINEUPID, :STATIONID, :CHANNEL, :CHANNELMINOR)");
00271 
00272         query.bindValue(":LINEUPID",    curr_lineupmap.lineupid);
00273         query.bindValue(":STATIONID",   curr_lineupmap.stationid);
00274         query.bindValue(":CHANNEL",     curr_lineupmap.channel);
00275         query.bindValue(":CHANNELMINOR",curr_lineupmap.channelMinor);
00276         if (!query.exec())
00277             MythDB::DBError("Inserting into dd_lineupmap", query);
00278     }
00279     else if (pqname == "schedule")
00280     {
00281         QDateTime endtime = curr_schedule.time.addSecs(
00282             QTime().secsTo(curr_schedule.duration));
00283 
00284         query.prepare(
00285             "INSERT INTO dd_schedule "
00286             "     ( programid,      stationid,   scheduletime,   "
00287             "       duration,       isrepeat,    stereo,         "
00288             "       dolby,          subtitled,   hdtv,           "
00289             "       closecaptioned, tvrating,    partnumber,      "
00290             "       parttotal,      endtime,     isnew) "
00291             "VALUES "
00292             "     (:PROGRAMID, :STATIONID,  :TIME,           "
00293             "      :DURATION,  :ISREPEAT,   :STEREO,         "
00294             "      :DOLBY,     :SUBTITLED,  :HDTV,           "
00295             "      :CAPTIONED, :TVRATING,   :PARTNUMBER,     "
00296             "      :PARTTOTAL, :ENDTIME,    :ISNEW)");
00297 
00298         query.bindValue(":PROGRAMID",   curr_schedule.programid);
00299         query.bindValue(":STATIONID",   curr_schedule.stationid);
00300         query.bindValue(":TIME",        curr_schedule.time);
00301         query.bindValue(":DURATION",    curr_schedule.duration);
00302         query.bindValue(":ISREPEAT",    curr_schedule.repeat);
00303         query.bindValue(":STEREO",      curr_schedule.stereo);
00304         query.bindValue(":DOLBY",       curr_schedule.dolby);
00305         query.bindValue(":SUBTITLED",   curr_schedule.subtitled);
00306         query.bindValue(":HDTV",        curr_schedule.hdtv);
00307         query.bindValue(":CAPTIONED",   curr_schedule.closecaptioned);
00308         query.bindValue(":TVRATING",    curr_schedule.tvrating);
00309         query.bindValue(":PARTNUMBER",  curr_schedule.partnumber);
00310         query.bindValue(":PARTTOTAL",   curr_schedule.parttotal);
00311         query.bindValue(":ENDTIME",     endtime);
00312         query.bindValue(":ISNEW",       curr_schedule.isnew);
00313 
00314         if (!query.exec())
00315             MythDB::DBError("Inserting into dd_schedule", query);
00316     }
00317     else if (pqname == "program")
00318     {
00319         float staravg = 0.0;
00320         if (!curr_program.starRating.isEmpty())
00321         {
00322             int fullstarcount = curr_program.starRating.count("*");
00323             int halfstarcount = curr_program.starRating.count("+");
00324             staravg = (fullstarcount + (halfstarcount * .5)) / 4;
00325         }
00326 
00327         QString cat_type = "";
00328         QString prefix = curr_program.programid.left(2);
00329 
00330         if (prefix == "MV")
00331             cat_type = "movie";
00332         else if (prefix == "SP")
00333             cat_type = "sports";
00334         else if (prefix == "EP" ||
00335                  curr_program.showtype.contains("series", Qt::CaseInsensitive))
00336             cat_type = "series";
00337         else
00338             cat_type = "tvshow";
00339 
00340         query.prepare(
00341             "INSERT INTO dd_program "
00342             "     ( programid,    title,       subtitle,       "
00343             "       description,  showtype,    category_type,  "
00344             "       mpaarating,   starrating,  stars,          "
00345             "       runtime,      year,        seriesid,       "
00346             "       colorcode,    syndicatedepisodenumber, originalairdate) "
00347             "VALUES "
00348             "     (:PROGRAMID,   :TITLE,      :SUBTITLE,       "
00349             "      :DESCRIPTION, :SHOWTYPE,   :CATTYPE,        "
00350             "      :MPAARATING,  :STARRATING, :STARS,          "
00351             "      :RUNTIME,     :YEAR,       :SERIESID,       "
00352             "      :COLORCODE,   :SYNDNUM,    :ORIGAIRDATE)    ");
00353 
00354         query.bindValue(":PROGRAMID",   curr_program.programid);
00355         query.bindValue(":TITLE",       curr_program.title);
00356         query.bindValue(":SUBTITLE",    curr_program.subtitle);
00357         query.bindValue(":DESCRIPTION", curr_program.description);
00358         query.bindValue(":SHOWTYPE",    curr_program.showtype);
00359         query.bindValue(":CATTYPE",     cat_type);
00360         query.bindValue(":MPAARATING",  curr_program.mpaaRating);
00361         query.bindValue(":STARRATING",  curr_program.starRating);
00362         query.bindValue(":STARS",       staravg);
00363         query.bindValue(":RUNTIME",     curr_program.duration);
00364         query.bindValue(":YEAR",        curr_program.year);
00365         query.bindValue(":SERIESID",    curr_program.seriesid);
00366         query.bindValue(":COLORCODE",   curr_program.colorcode);
00367         query.bindValue(":SYNDNUM",     curr_program.syndicatedEpisodeNumber);
00368         query.bindValue(":ORIGAIRDATE", curr_program.originalAirDate);
00369 
00370         if (!query.exec())
00371             MythDB::DBError("Inserting into dd_program", query);
00372     }
00373     else if (pqname == "member")
00374     {
00375         QString roleunderlines = curr_productioncrew.role.replace(" ", "_");
00376 
00377         QString fullname = curr_productioncrew.givenname;
00378         if (!fullname.isEmpty())
00379             fullname += " ";
00380         fullname += curr_productioncrew.surname;
00381 
00382         query.prepare(
00383             "INSERT INTO dd_productioncrew "
00384             "       ( programid,  role,  givenname,  surname,  fullname) "
00385             "VALUES (:PROGRAMID, :ROLE, :GIVENNAME, :SURNAME, :FULLNAME)");
00386 
00387         query.bindValue(":PROGRAMID",   lastprogramid);
00388         query.bindValue(":ROLE",        roleunderlines);
00389         query.bindValue(":GIVENNAME",   curr_productioncrew.givenname);
00390         query.bindValue(":SURNAME",     curr_productioncrew.surname);
00391         query.bindValue(":FULLNAME",    fullname);
00392 
00393         if (!query.exec())
00394             MythDB::DBError("Inserting into dd_productioncrew", query);
00395 
00396         curr_productioncrew.givenname = "";
00397         curr_productioncrew.surname = "";
00398     }
00399     else if (pqname == "genre")
00400     {
00401         query.prepare(
00402             "INSERT INTO dd_genre "
00403             "       ( programid,  class,  relevance) "
00404             "VALUES (:PROGRAMID, :CLASS, :RELEVANCE)");
00405 
00406         query.bindValue(":PROGRAMID",   lastprogramid);
00407         query.bindValue(":CLASS",       curr_genre.gclass);
00408         query.bindValue(":RELEVANCE",   curr_genre.relevance);
00409 
00410         if (!query.exec())
00411             MythDB::DBError("Inserting into dd_genre", query);
00412     }
00413 
00414     return true;
00415 }
00416 
00417 bool DDStructureParser::startDocument()
00418 {
00419     parent.CreateTempTables();
00420     return true;
00421 }
00422 
00423 bool DDStructureParser::endDocument()
00424 {
00425     return true;
00426 }
00427 
00428 bool DDStructureParser::characters(const QString& pchars)
00429 {
00430 #if 0
00431     LOG(VB_GENERAL, LOG_DEBUG, LOC + "Characters : " + pchars);
00432 #endif
00433     if (pchars.trimmed().isEmpty())
00434         return true;
00435 
00436     if (currtagname == "message")
00437     {
00438         if (pchars.contains("expire"))
00439         {
00440             QString ExtractDateFromMessage = pchars.right(20);
00441             QDateTime EDFM = QDateTime::fromString(ExtractDateFromMessage,
00442                                                    Qt::ISODate);
00443             QString SDDateFormat = GetMythDB()->GetSetting("DateFormat",
00444                                                            "ddd d MMMM");
00445             // Ensure we show the year when it's important, regardless of
00446             // specified DateFormat
00447             if ((!SDDateFormat.contains('y')) &&
00448                 (EDFM.date().year() != QDate::currentDate().year()))
00449             {
00450                 SDDateFormat.append(" (yyyy)");
00451             }
00452             QString dateFormat = QString("%1 %2")
00453                     .arg(SDDateFormat)
00454                     .arg(GetMythDB()->GetSetting("TimeFormat", "hh:mm"));
00455             QString ExpirationDate = EDFM.toString(dateFormat);
00456 
00457             QString ExpirationDateMessage = "Your subscription expires on " +
00458                 ExpirationDate;
00459 
00460             QDateTime curTime = QDateTime::currentDateTime();
00461             if (curTime.daysTo(EDFM) <= 5)
00462             {
00463                 LOG(VB_GENERAL, LOG_WARNING, LOC + QString("WARNING: ") +
00464                         ExpirationDateMessage);
00465             }
00466             else
00467             {
00468                 LOG(VB_GENERAL, LOG_INFO, LOC + ExpirationDateMessage);
00469             }
00470 
00471             MSqlQuery query(MSqlQuery::DDCon());
00472 
00473             QString querystr = QString(
00474                 "UPDATE settings "
00475                 "SET data ='%1' "
00476                 "WHERE value='DataDirectMessage'")
00477                 .arg(ExpirationDateMessage);
00478 
00479             query.prepare(querystr);
00480 
00481             if (!query.exec())
00482             {
00483                 MythDB::DBError("Updating DataDirect Status Message",
00484                                      query);
00485             }
00486         }
00487     }
00488     if (currtagname == "callSign")
00489         curr_station.callsign = pchars;
00490     else if (currtagname == "name")
00491         curr_station.stationname = pchars;
00492     else if (currtagname == "affiliate")
00493         curr_station.affiliate = pchars;
00494     else if (currtagname == "fccChannelNumber")
00495         curr_station.fccchannelnumber = pchars;
00496     else if (currtagname == "title")
00497         curr_program.title = pchars;
00498     else if (currtagname == "subtitle")
00499         curr_program.subtitle = pchars;
00500     else if (currtagname == "description")
00501         curr_program.description = pchars;
00502     else if (currtagname == "showType")
00503         curr_program.showtype = pchars;
00504     else if (currtagname == "series")
00505         curr_program.seriesid = pchars;
00506     else if (currtagname == "colorCode")
00507         curr_program.colorcode = pchars;
00508     else if (currtagname == "mpaaRating")
00509         curr_program.mpaaRating = pchars;
00510     else if (currtagname == "starRating")
00511         curr_program.starRating = pchars;
00512     else if (currtagname == "year")
00513         curr_program.year = pchars;
00514     else if (currtagname == "syndicatedEpisodeNumber")
00515         curr_program.syndicatedEpisodeNumber = pchars;
00516     else if (currtagname == "runTime")
00517     {
00518         QString runtimestr = pchars;
00519         QTime runtime = QTime(runtimestr.mid(2,2).toInt(),
00520                               runtimestr.mid(5,2).toInt(), 0, 0);
00521         curr_program.duration = runtime;
00522     }
00523     else if (currtagname == "originalAirDate")
00524     {
00525         QDate airdate = QDate::fromString(pchars, Qt::ISODate);
00526         curr_program.originalAirDate = airdate;
00527     }
00528     else if (currtagname == "role")
00529         curr_productioncrew.role = pchars;
00530     else if (currtagname == "givenname")
00531         curr_productioncrew.givenname = pchars;
00532     else if (currtagname == "surname")
00533         curr_productioncrew.surname = pchars;
00534     else if (currtagname == "class")
00535         curr_genre.gclass = pchars;
00536     else if (currtagname == "relevance")
00537         curr_genre.relevance = pchars;
00538 
00539     return true;
00540 }
00541 
00542 DataDirectProcessor::DataDirectProcessor(uint lp, QString user, QString pass) :
00543     m_listingsProvider(lp % DD_PROVIDER_COUNT),
00544     m_userid(user),                   m_password(pass),
00545     m_tmpDir("/tmp"),                 m_cacheData(false),
00546     m_inputFilename(""),              m_tmpPostFile(QString::null),
00547     m_tmpResultFile(QString::null),   m_cookieFile(QString::null),
00548     m_cookieFileDT()
00549 {
00550     {
00551         QMutexLocker locker(&user_agent_lock);
00552         user_agent = QString("MythTV/%1.%2")
00553             .arg(MYTH_BINARY_VERSION).arg(MYTH_SOURCE_VERSION);
00554     }
00555 
00556     DataDirectURLs urls0(
00557         "Tribune Media Zap2It",
00558         "http://datadirect.webservices.zap2it.com/tvlistings/xtvdService",
00559         "http://labs.zap2it.com",
00560         "/ztvws/ztvws_login/1,1059,TMS01-1,00.html");
00561     DataDirectURLs urls1(
00562         "Schedules Direct",
00563         "http://webservices.schedulesdirect.tmsdatadirect.com"
00564         "/schedulesdirect/tvlistings/xtvdService",
00565         "http://schedulesdirect.org",
00566         "/login/index.php");
00567     m_providers.push_back(urls0);
00568     m_providers.push_back(urls1);
00569 }
00570 
00571 DataDirectProcessor::~DataDirectProcessor()
00572 {
00573     LOG(VB_GENERAL, LOG_INFO, LOC + "Deleting temporary files");
00574 
00575     if (!m_tmpPostFile.isEmpty())
00576     {
00577         QByteArray tmp = m_tmpPostFile.toAscii();
00578         unlink(tmp.constData());
00579     }
00580 
00581     if (!m_tmpResultFile.isEmpty())
00582     {
00583         QByteArray tmp = m_tmpResultFile.toAscii();
00584         unlink(tmp.constData());
00585     }
00586 
00587     if (!m_cookieFile.isEmpty())
00588     {
00589         QByteArray tmp = m_cookieFile.toAscii();
00590         unlink(tmp.constData());
00591     }
00592 
00593     QDir d(m_tmpDir, "mythtv_dd_cache_*", QDir::Name,
00594            QDir::Files | QDir::NoSymLinks);
00595 
00596     for (uint i = 0; i < d.count(); i++)
00597     {
00598         QString    tmps = QString(m_tmpDir + "/" + d[i]);
00599         QByteArray tmpa = tmps.toAscii();
00600         unlink(tmpa.constData());
00601     }
00602 
00603     if (m_tmpDir != "/tmp")
00604     {
00605         QByteArray tmp = m_tmpDir.toAscii();
00606         rmdir(tmp.constData());
00607     }
00608 }
00609 
00610 void DataDirectProcessor::UpdateStationViewTable(QString lineupid)
00611 {
00612     MSqlQuery query(MSqlQuery::DDCon());
00613 
00614     if (!query.exec("TRUNCATE TABLE dd_v_station;"))
00615         MythDB::DBError("Truncating temporary table dd_v_station", query);
00616 
00617     query.prepare(
00618         "INSERT INTO dd_v_station "
00619         "     ( stationid,            callsign,         stationname, "
00620         "       affiliate,            fccchannelnumber, channel,     "
00621         "       channelMinor) "
00622         "SELECT dd_station.stationid, callsign,         stationname, "
00623         "       affiliate,            fccchannelnumber, channel,     "
00624         "       channelMinor "
00625         "FROM dd_station, dd_lineupmap "
00626         "WHERE ((dd_station.stationid  = dd_lineupmap.stationid) AND "
00627         "       (dd_lineupmap.lineupid = :LINEUP))");
00628 
00629     query.bindValue(":LINEUP", lineupid);
00630 
00631     if (!query.exec())
00632         MythDB::DBError("Populating temporary table dd_v_station", query);
00633 }
00634 
00635 void DataDirectProcessor::UpdateProgramViewTable(uint sourceid)
00636 {
00637     MSqlQuery query(MSqlQuery::DDCon());
00638 
00639     if (!query.exec("TRUNCATE TABLE dd_v_program;"))
00640         MythDB::DBError("Truncating temporary table dd_v_program", query);
00641 
00642     QString qstr =
00643         "INSERT INTO dd_v_program "
00644         "     ( chanid,                  starttime,       endtime,         "
00645         "       title,                   subtitle,        description,     "
00646         "       airdate,                 stars,           previouslyshown, "
00647         "       stereo,                  dolby,           subtitled,       "
00648         "       hdtv,                    closecaptioned,  partnumber,      "
00649         "       parttotal,               seriesid,        originalairdate, "
00650         "       showtype,                category_type,   colorcode,       "
00651         "       syndicatedepisodenumber, tvrating,        mpaarating,      "
00652         "       programid )      "
00653         "SELECT chanid,                  scheduletime,    endtime,         "
00654         "       title,                   subtitle,        description,     "
00655         "       year,                    stars,           isrepeat,        "
00656         "       stereo,                  dolby,           subtitled,       "
00657         "       hdtv,                    closecaptioned,  partnumber,      "
00658         "       parttotal,               seriesid,        originalairdate, "
00659         "       showtype,                category_type,   colorcode,       "
00660         "       syndicatedepisodenumber, tvrating,        mpaarating,      "
00661         "       dd_program.programid "
00662         "FROM channel, dd_schedule, dd_program "
00663         "WHERE ((dd_schedule.programid = dd_program.programid)  AND "
00664         "       (channel.xmltvid       = dd_schedule.stationid) AND "
00665         "       (channel.sourceid      = :SOURCEID))";
00666 
00667     query.prepare(qstr);
00668 
00669     query.bindValue(":SOURCEID", sourceid);
00670 
00671     if (!query.exec())
00672         MythDB::DBError("Populating temporary table dd_v_program", query);
00673 
00674     if (!query.exec("ANALYZE TABLE dd_v_program;"))
00675         MythDB::DBError("Analyzing table dd_v_program", query);
00676 
00677     if (!query.exec("ANALYZE TABLE dd_productioncrew;"))
00678         MythDB::DBError("Analyzing table dd_productioncrew", query);
00679 }
00680 
00681 int DataDirectProcessor::UpdateChannelsSafe(
00682     uint sourceid,
00683     bool insert_channels,
00684     bool filter_new_channels)
00685 {
00686     int new_channels = 0;
00687 
00688     if (!SourceUtil::GetConnectionCount(sourceid))
00689     {
00690         LOG(VB_GENERAL, LOG_WARNING, LOC +
00691             QString("Not inserting channels into disconnected source %1.")
00692                 .arg(sourceid));
00693         return -1;
00694     }
00695 
00696     if (!SourceUtil::IsProperlyConnected(sourceid, true))
00697         return -1;
00698 
00699     // Find all the channels in the dd_v_station temp table
00700     // where there is no channel with the same xmltvid in the
00701     // DB using the same source.
00702     MSqlQuery query(MSqlQuery::DDCon());
00703     query.prepare(
00704         "SELECT dd_v_station.stationid,   dd_v_station.callsign,         "
00705         "       dd_v_station.stationname, dd_v_station.fccchannelnumber, "
00706         "       dd_v_station.channel,     dd_v_station.channelMinor      "
00707         "FROM dd_v_station LEFT JOIN channel ON "
00708         "     dd_v_station.stationid = channel.xmltvid AND "
00709         "     channel.sourceid = :SOURCEID "
00710         "WHERE channel.chanid IS NULL");
00711     query.bindValue(":SOURCEID", sourceid);
00712 
00713     if (!query.exec())
00714     {
00715         MythDB::DBError("Selecting new channels", query);
00716         return -1;
00717     }
00718 
00719     bool is_encoder = (SourceUtil::IsCableCardPresent(sourceid) ||
00720                        SourceUtil::IsEncoder(sourceid, true) ||
00721                        SourceUtil::IsUnscanable(sourceid));
00722 
00723     while (query.next())
00724     {
00725         QString xmltvid    = query.value(0).toString();
00726         QString callsign   = query.value(1).toString();
00727         QString name       = query.value(2).toString();
00728         uint    freqid     = query.value(3).toUInt();
00729         QString chan_major = query.value(4).toString();
00730         QString chan_minor = query.value(5).toString();
00731 
00732         if (filter_new_channels && is_encoder &&
00733             (query.value(5).toUInt() > 0))
00734         {
00735 #if 0
00736             LOG(VB_GENERAL, LOG_INFO, LOC +
00737                 QString("Not adding channel %1-%2 '%3' (%4),\n\t\t\t"
00738                         "looks like a digital channel on an analog source.")
00739                     .arg(chan_major).arg(chan_minor).arg(name).arg(callsign));
00740 #endif
00741             continue;
00742         }
00743 
00744         uint mods =
00745             update_channel_basic(sourceid, insert_channels && is_encoder,
00746                                  xmltvid, callsign, name, freqid,
00747                                  chan_major, chan_minor);
00748 
00749         (void) mods;
00750 #if 0
00751         if (!insert_channels && !mods)
00752         {
00753             LOG(VB_GENERAL, LOG_INFO, LOC +
00754                 QString("Not adding channel '%1' (%2).")
00755                     .arg(name).arg(callsign));
00756         }
00757 #endif
00758         new_channels++;
00759     }
00760 
00761     teardown_frequency_tables();
00762 
00763     return new_channels;
00764 }
00765 
00766 bool DataDirectProcessor::UpdateChannelsUnsafe(
00767     uint sourceid, bool filter_new_channels)
00768 {
00769     if (filter_new_channels &&
00770         !SourceUtil::IsProperlyConnected(sourceid, false))
00771     {
00772         return false;
00773     }
00774 
00775     MSqlQuery dd_station_info(MSqlQuery::DDCon());
00776     dd_station_info.prepare(
00777         "SELECT callsign,         stationname, stationid,"
00778         "       fccchannelnumber, channel,     channelMinor "
00779         "FROM dd_v_station");
00780     if (!dd_station_info.exec())
00781         return false;
00782 
00783     if (dd_station_info.size() == 0)
00784         return true;
00785 
00786     MSqlQuery chan_update_q(MSqlQuery::DDCon());
00787     chan_update_q.prepare(
00788         "UPDATE channel "
00789         "SET callsign  = :CALLSIGN,  name   = :NAME, "
00790         "    channum   = :CHANNUM,   freqid = :FREQID, "
00791         "    atsc_major_chan = :MAJORCHAN, "
00792         "    atsc_minor_chan = :MINORCHAN "
00793         "WHERE xmltvid = :STATIONID AND sourceid = :SOURCEID");
00794 
00795     bool is_encoder = (SourceUtil::IsCableCardPresent(sourceid) ||
00796                        SourceUtil::IsEncoder(sourceid, true) ||
00797                        SourceUtil::IsUnscanable(sourceid));
00798 
00799     while (dd_station_info.next())
00800     {
00801         uint    freqid     = dd_station_info.value(3).toUInt();
00802         QString chan_major = dd_station_info.value(4).toString();
00803         QString chan_minor = dd_station_info.value(5).toString();
00804         QString tvformat   = QString::null;
00805         QString channum    = process_dd_station(
00806             sourceid, chan_major, chan_minor, tvformat, freqid);
00807 
00808         if (filter_new_channels && is_encoder &&
00809             (dd_station_info.value(5).toUInt() > 0))
00810         {
00811 #if 0
00812             LOG(VB_GENERAL, LOG_INFO, LOC +
00813                 QString("Not adding channel %1-%2 '%3' (%4),\n\t\t\t"
00814                         "looks like a digital channel on an analog source.")
00815                     .arg(chan_major).arg(chan_minor)
00816                     .arg(dd_station_info.value(1).toString())
00817                     .arg(dd_station_info.value(0).toString()));
00818 #endif
00819             continue;
00820         }
00821 
00822         chan_update_q.bindValue(":CALLSIGN",  dd_station_info.value(0));
00823         chan_update_q.bindValue(":NAME",      dd_station_info.value(1));
00824         chan_update_q.bindValue(":STATIONID", dd_station_info.value(2));
00825         chan_update_q.bindValue(":CHANNUM",   channum);
00826         chan_update_q.bindValue(":SOURCEID",  sourceid);
00827         chan_update_q.bindValue(":FREQID",    freqid);
00828         chan_update_q.bindValue(":MAJORCHAN", chan_major.toUInt());
00829         chan_update_q.bindValue(":MINORCHAN", chan_minor.toUInt());
00830 
00831         if (!chan_update_q.exec())
00832         {
00833             MythDB::DBError("Updating channel table", chan_update_q);
00834         }
00835     }
00836 
00837     return true;
00838 }
00839 
00840 void DataDirectProcessor::DataDirectProgramUpdate(void)
00841 {
00842     MSqlQuery query(MSqlQuery::DDCon());
00843 
00844 #if 0
00845     LOG(VB_GENERAL, LOG_DEBUG, LOC +
00846         "Adding rows to main program table from view table");
00847 #endif
00848     query.prepare(
00849         "INSERT IGNORE INTO program "
00850         "  ( chanid,        starttime,   endtime,         title,           "
00851         "    subtitle,      description, showtype,        category,        "
00852         "    category_type, airdate,     stars,           previouslyshown, "
00853         "    stereo,        subtitled,   subtitletypes,   videoprop,       "
00854         "    audioprop,     hdtv,        closecaptioned,  partnumber,      "
00855         "    parttotal,     seriesid,    originalairdate, colorcode,       "
00856         "    syndicatedepisodenumber,                                      "
00857         "                   programid,   listingsource)                    "
00858         "  SELECT                                                          "
00859         "    dd_v_program.chanid,                                          "
00860         "    DATE_ADD(starttime, INTERVAL channel.tmoffset MINUTE),        "
00861         "    DATE_ADD(endtime, INTERVAL channel.tmoffset MINUTE),          "
00862         "                                                 title,           "
00863         "    subtitle,      description, showtype,        dd_genre.class,  "
00864         "    category_type, airdate,     stars,           previouslyshown, "
00865         "    stereo,        subtitled,                                     "
00866         "    (subtitled << 1 ) | closecaptioned, hdtv,                     "
00867         "    (dolby << 3) | stereo,                                        "
00868         "                   hdtv,        closecaptioned,  partnumber,      "
00869         "    parttotal,     seriesid,    originalairdate, colorcode,       "
00870         "    syndicatedepisodenumber,                                      "
00871         "                   dd_v_program.programid,                        "
00872         "                               :LSOURCE                           "
00873         "FROM (dd_v_program, channel) "
00874         "LEFT JOIN dd_genre ON "
00875         "  ( dd_v_program.programid = dd_genre.programid AND  "
00876         "    dd_genre.relevance     = '0' ) "
00877         "WHERE dd_v_program.chanid = channel.chanid");
00878 
00879     query.bindValue(":LSOURCE", kListingSourceDDSchedulesDirect);
00880 
00881     if (!query.exec())
00882         MythDB::DBError("Inserting into program table", query);
00883 
00884 #if 0
00885     LOG(VB_GENERAL, LOG_DEBUG, LOC +
00886         "Finished adding rows to main program table");
00887     LOG(VB_GENERAL, LOG_DEBUG, LOC + "Adding program ratings");
00888 #endif
00889 
00890     if (!query.exec("INSERT IGNORE INTO programrating (chanid, starttime, "
00891                     "system, rating) SELECT dd_v_program.chanid, "
00892                     "DATE_ADD(starttime, INTERVAL channel.tmoffset MINUTE), "
00893                     " 'MPAA', "
00894                     "mpaarating FROM dd_v_program, channel WHERE "
00895                     "mpaarating != '' AND dd_v_program.chanid = "
00896                     "channel.chanid"))
00897         MythDB::DBError("Inserting into programrating table", query);
00898 
00899     if (!query.exec("INSERT IGNORE INTO programrating (chanid, starttime, "
00900                     "system, rating) SELECT dd_v_program.chanid, "
00901                     "DATE_ADD(starttime, INTERVAL channel.tmoffset MINUTE), "
00902                     "'VCHIP', "
00903                     "tvrating FROM dd_v_program, channel WHERE tvrating != ''"
00904                     " AND dd_v_program.chanid = channel.chanid"))
00905         MythDB::DBError("Inserting into programrating table", query);
00906 
00907 #if 0
00908     LOG(VB_GENERAL, LOG_DEBUG, LOC + "Finished adding program ratings");
00909     LOG(VB_GENERAL, LOG_DEBUG, LOC +
00910         "Populating people table from production crew list");
00911 #endif
00912 
00913     if (!query.exec("INSERT IGNORE INTO people (name) "
00914                     "SELECT fullname "
00915                     "FROM dd_productioncrew "
00916                     "LEFT OUTER JOIN people "
00917                     "ON people.name = dd_productioncrew.fullname "
00918                     "WHERE people.name IS NULL;"))
00919         MythDB::DBError("Inserting into people table", query);
00920 
00921 #if 0
00922     LOG(VB_GENERAL, LOG_INFO, LOC + "Finished adding people");
00923     LOG(VB_GENERAL, LOG_INFO, LOC +
00924         "Adding credits entries from production crew list");
00925 #endif
00926 
00927     if (!query.exec("INSERT IGNORE INTO credits (chanid, starttime, person, role)"
00928                     "SELECT dd_v_program.chanid, "
00929                     "DATE_ADD(dd_v_program.starttime, INTERVAL channel.tmoffset MINUTE), "
00930                     "people.person, "
00931                     "dd_productioncrew.role "
00932                     "FROM dd_v_program "
00933                     "JOIN channel "
00934                     "ON dd_v_program.chanid = channel.chanid "
00935                     "JOIN dd_productioncrew "
00936                     "ON dd_productioncrew.programid = dd_v_program.programid "
00937                     "JOIN people "
00938                     "ON people.name = dd_productioncrew.fullname "
00939                     "LEFT OUTER JOIN credits "
00940                     "ON credits.chanid = dd_v_program.chanid "
00941                     "AND credits.starttime = DATE_ADD(dd_v_program.starttime, INTERVAL channel.tmoffset MINUTE) "
00942                     "AND credits.person = people.person "
00943                     "AND credits.role = dd_productioncrew.role "
00944                     "WHERE credits.role IS NULL;"))
00945         MythDB::DBError("Inserting into credits table", query);
00946 
00947 #if 0
00948     LOG(VB_GENERAL, LOG_DEBUG, LOC + "Finished inserting credits");
00949     LOG(VB_GENERAL, LOG_DEBUG, LOC + "Adding genres");
00950 #endif
00951 
00952     if (!query.exec("INSERT IGNORE INTO programgenres (chanid, starttime, "
00953                     "relevance, genre) SELECT dd_v_program.chanid, "
00954                     "DATE_ADD(starttime, INTERVAL channel.tmoffset MINUTE), "
00955                     "relevance, class FROM dd_v_program, dd_genre, channel "
00956                     "WHERE (dd_v_program.programid = dd_genre.programid) "
00957                     "AND dd_v_program.chanid = channel.chanid"))
00958         MythDB::DBError("Inserting into programgenres table",query);
00959 
00960 #if 0
00961     LOG(VB_GENERAL, LOG_DEBUG, LOC + "Done");
00962 #endif
00963 }
00964 
00965 void authenticationCallback(QNetworkReply *reply, QAuthenticator *auth,
00966                             void *arg)
00967 {
00968     if (!arg)
00969         return;
00970 
00971     DataDirectProcessor *dd = reinterpret_cast<DataDirectProcessor *>(arg);
00972     dd->authenticationCallback(reply, auth);
00973 }
00974 
00975 void DataDirectProcessor::authenticationCallback(QNetworkReply *reply,
00976                                                  QAuthenticator *auth)
00977 {
00978     LOG(VB_FILE, LOG_DEBUG, "DataDirect auth callback");
00979     (void)reply;
00980     auth->setUser(GetUserID());
00981     auth->setPassword(GetPassword());
00982 }
00983 
00984 bool DataDirectProcessor::DDPost(QString    ddurl,        QString   &inputFile,
00985                                  QDateTime  pstartDate,   QDateTime  pendDate,
00986                                  QString   &err_txt)
00987 {
00988     if (!inputFile.isEmpty() && QFile(inputFile).exists())
00989     {
00990         return true;
00991     }
00992 
00993     QString startdatestr = pstartDate.toString(Qt::ISODate) + "Z";
00994     QString enddatestr = pendDate.toString(Qt::ISODate) + "Z";
00995     QByteArray postdata;
00996     postdata  = "<?xml version='1.0' encoding='utf-8'?>\n";
00997     postdata += "<SOAP-ENV:Envelope\n";
00998     postdata += "xmlns:SOAP-ENV='http://schemas.xmlsoap.org/soap/envelope/'\n";
00999     postdata += "xmlns:xsd='http://www.w3.org/2001/XMLSchema'\n";
01000     postdata += "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n";
01001     postdata += "xmlns:SOAP-ENC='http://schemas.xmlsoap.org/soap/encoding/'>\n";
01002     postdata += "<SOAP-ENV:Body>\n";
01003     postdata += "<ns1:download  xmlns:ns1='urn:TMSWebServices'>\n";
01004     postdata += "<startTime xsi:type='xsd:dateTime'>";
01005     postdata += startdatestr;
01006     postdata += "</startTime>\n";
01007     postdata += "<endTime xsi:type='xsd:dateTime'>";
01008     postdata += enddatestr;
01009     postdata += "</endTime>\n";
01010     postdata += "</ns1:download>\n";
01011     postdata += "</SOAP-ENV:Body>\n";
01012     postdata += "</SOAP-ENV:Envelope>\n";
01013 
01014     if (inputFile.isEmpty()) {
01015         inputFile = QString("/tmp/mythtv_ddp_data");
01016     }
01017 
01018     const QByteArray header = "Accept-Encoding";
01019     const QByteArray value  = "gzip";
01020 
01021     LOG(VB_GENERAL, LOG_INFO, "Downloading DataDirect feed");
01022 
01023     MythDownloadManager *manager = GetMythDownloadManager();
01024 
01025     if (!manager->postAuth(ddurl, &postdata, &::authenticationCallback, this,
01026                            &header, &value))
01027     {
01028         err_txt = QString("Download error");
01029         return false;
01030     }
01031 
01032     LOG(VB_GENERAL, LOG_INFO, QString("Downloaded %1 bytes")
01033         .arg(postdata.size()));
01034 
01035     LOG(VB_GENERAL, LOG_INFO, "Uncompressing DataDirect feed");
01036 
01037     QByteArray uncompressed = gUncompress(postdata);
01038 
01039     LOG(VB_GENERAL, LOG_INFO, QString("Uncompressed to %1 bytes")
01040         .arg(uncompressed.size()));
01041 
01042     if (uncompressed.size() == 0)
01043         uncompressed = postdata;
01044 
01045     QFile file(inputFile);
01046     file.open(QIODevice::WriteOnly);
01047     file.write(uncompressed);
01048     file.close();
01049 
01050     if (uncompressed.size() == 0)
01051     {
01052         err_txt = QString("Error uncompressing data");
01053         return false;
01054     }
01055 
01056     return true;
01057 }
01058 
01059 bool DataDirectProcessor::GrabNextSuggestedTime(void)
01060 {
01061     LOG(VB_GENERAL, LOG_INFO, LOC + "Grabbing next suggested grabbing time");
01062 
01063     QString ddurl = m_providers[m_listingsProvider].webServiceURL;
01064 
01065     bool ok;
01066     QString resultFilename = GetResultFilename(ok);
01067     if (!ok)
01068     {
01069         LOG(VB_GENERAL, LOG_ERR, LOC +
01070             "GrabNextSuggestedTime: Creating temp result file");
01071         return false;
01072     }
01073 
01074     QByteArray postdata;
01075     postdata  = "<?xml version='1.0' encoding='utf-8'?>\n";
01076     postdata += "<SOAP-ENV:Envelope\n";
01077     postdata += "xmlns:SOAP-ENV='http://schemas.xmlsoap.org/soap/envelope/'\n";
01078     postdata += "xmlns:xsd='http://www.w3.org/2001/XMLSchema'\n";
01079     postdata += "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n";
01080     postdata += "xmlns:SOAP-ENC='http://schemas.xmlsoap.org/soap/encoding/'>\n";
01081     postdata += "<SOAP-ENV:Body>\n";
01082     postdata += "<tms:acknowledge xmlns:tms='urn:TMSWebServices'>\n";
01083     postdata += "</SOAP-ENV:Body>\n";
01084     postdata += "</SOAP-ENV:Envelope>\n";
01085 
01086     MythDownloadManager *manager = GetMythDownloadManager();
01087 
01088     if (!manager->postAuth(ddurl, &postdata, &::authenticationCallback, this))
01089     {
01090         LOG(VB_GENERAL, LOG_ERR, LOC +
01091             "GrabNextSuggestedTime: Could not download");
01092         return false;
01093     }
01094 
01095     QDateTime NextSuggestedTime;
01096     QDateTime BlockedTime;
01097 
01098     LOG(VB_GENERAL, LOG_INFO, QString("Suggested Time data: %1 bytes")
01099         .arg(postdata.size()));
01100 
01101     QFile file(resultFilename);
01102     file.open(QIODevice::WriteOnly);
01103     file.write(postdata);
01104     file.close();
01105 
01106     bool GotNextSuggestedTime = false;
01107     MUNUSED bool GotBlockedTime = false;
01108 
01109     if (file.open(QIODevice::ReadOnly))
01110     {
01111         QTextStream stream(&file);
01112         QString line;
01113         while (!stream.atEnd())
01114         {
01115             line = stream.readLine();
01116             if (line.contains("<suggestedTime>", Qt::CaseInsensitive))
01117             {
01118                 QString tmpStr = line;
01119                 tmpStr.replace(
01120                     QRegExp(".*<suggestedTime>([^<]*)</suggestedTime>.*"),
01121                     "\\1");
01122 
01123                 GotNextSuggestedTime = true;
01124                 QDateTime UTCdt = QDateTime::fromString(tmpStr, Qt::ISODate);
01125                 NextSuggestedTime = MythUTCToLocal(UTCdt);
01126                 LOG(VB_GENERAL, LOG_INFO, LOC +
01127                     QString("NextSuggestedTime is: ") +
01128                     NextSuggestedTime.toString(Qt::ISODate));
01129             }
01130 
01131             if (line.contains("<blockedTime>", Qt::CaseInsensitive))
01132             {
01133                 QString tmpStr = line;
01134                 tmpStr.replace(
01135                     QRegExp(".*<blockedTime>([^<]*)</blockedTime>.*"), "\\1");
01136 
01137                 GotBlockedTime = true;
01138                 QDateTime UTCdt = QDateTime::fromString(tmpStr, Qt::ISODate);
01139                 BlockedTime = MythUTCToLocal(UTCdt);
01140                 LOG(VB_GENERAL, LOG_INFO, LOC + QString("BlockedTime is: ")
01141                         + BlockedTime.toString(Qt::ISODate));
01142             }
01143         }
01144         file.close();
01145     }
01146 
01147     if (GotNextSuggestedTime)
01148         gCoreContext->SaveSettingOnHost("MythFillSuggestedRunTime",
01149             NextSuggestedTime.toString(Qt::ISODate), NULL);
01150 
01151     return GotNextSuggestedTime;
01152 }
01153 
01154 bool DataDirectProcessor::GrabData(const QDateTime &pstartDate,
01155                                    const QDateTime &pendDate)
01156 {
01157     QString msg = (pstartDate.addSecs(1) == pendDate) ? "channel" : "listing";
01158     LOG(VB_GENERAL, LOG_INFO, LOC + "Grabbing " + msg + " data");
01159 
01160     QString err = "";
01161     QString ddurl = m_providers[m_listingsProvider].webServiceURL;
01162     QString inputfile = m_inputFilename;
01163     QString cache_dd_data = QString::null;
01164 
01165     if (m_cacheData)
01166     {
01167         QByteArray userid = GetUserID().toAscii();
01168         cache_dd_data = m_tmpDir + QString("/mythtv_dd_cache_%1_%2_UTC_%3_to_%4")
01169             .arg(GetListingsProvider())
01170             .arg(userid.constData())
01171             .arg(pstartDate.toString("yyyyMMddhhmmss"))
01172             .arg(pendDate.toString("yyyyMMddhhmmss"));
01173 
01174         if (QFile(cache_dd_data).exists() && m_inputFilename.isEmpty())
01175         {
01176             LOG(VB_GENERAL, LOG_INFO, LOC + "Using DD cache");
01177         }
01178 
01179         if( m_inputFilename.isEmpty() )
01180             inputfile = cache_dd_data;
01181     }
01182 
01183     if (!DDPost(ddurl, inputfile, pstartDate, pendDate, err))
01184     {
01185         LOG(VB_GENERAL, LOG_ERR, LOC + QString("Failed to get data: %1")
01186                 .arg(err));
01187         return false;
01188     }
01189 
01190     QFile file(inputfile);
01191     file.open(QIODevice::ReadOnly);
01192     QByteArray data = file.readAll();
01193     file.close();
01194 
01195     if (data.isEmpty())
01196     {
01197         LOG(VB_GENERAL, LOG_ERR, LOC + "Data is empty");
01198         return false;
01199     }
01200 
01201     bool ok = true;
01202 
01203     DDStructureParser ddhandler(*this);
01204     QXmlInputSource  xmlsource;
01205     QXmlSimpleReader xmlsimplereader;
01206 
01207     xmlsource.setData(data);
01208     xmlsimplereader.setContentHandler(&ddhandler);
01209     if (!xmlsimplereader.parse(xmlsource))
01210     {
01211         LOG(VB_GENERAL, LOG_ERR, LOC +
01212             "DataDirect XML failed to properly parse, downloaded listings "
01213             "were probably corrupt.");
01214         ok = false;
01215     }
01216 
01217     return ok;
01218 }
01219 
01220 bool DataDirectProcessor::GrabLineupsOnly(void)
01221 {
01222     const QDateTime start = QDateTime(QDate::currentDate().addDays(2),
01223                                       QTime(23, 59, 0));
01224     const QDateTime end   = start.addSecs(1);
01225 
01226     return GrabData(start, end);
01227 }
01228 
01229 bool DataDirectProcessor::GrabAllData(void)
01230 {
01231     return GrabData(QDateTime(QDate::currentDate()).addDays(-2),
01232                     QDateTime(QDate::currentDate()).addDays(15));
01233 }
01234 
01235 void DataDirectProcessor::CreateATempTable(const QString &ptablename,
01236                                            const QString &ptablestruct)
01237 {
01238     MSqlQuery query(MSqlQuery::DDCon());
01239     QString querystr;
01240     querystr = "CREATE TEMPORARY TABLE IF NOT EXISTS " + ptablename + " " +
01241         ptablestruct + " ENGINE=MyISAM;";
01242 
01243     if (!query.exec(querystr))
01244         MythDB::DBError("Creating temporary table", query);
01245 
01246     querystr = "TRUNCATE TABLE " + ptablename + ";";
01247 
01248     if (!query.exec(querystr))
01249         MythDB::DBError("Truncating temporary table", query);
01250 }
01251 
01252 void DataDirectProcessor::CreateTempTables()
01253 {
01254     QMap<QString,QString> dd_tables;
01255 
01256     dd_tables["dd_station"] =
01257         "( stationid char(12),           callsign char(10),     "
01258         "  stationname varchar(40),      affiliate varchar(25), "
01259         "  fccchannelnumber char(15) )";
01260 
01261     dd_tables["dd_lineup"] =
01262         "( lineupid char(100),           name char(42),  "
01263         "  type char(20),                postal char(6), "
01264         "  device char(30) )";
01265 
01266     dd_tables["dd_lineupmap"] =
01267         "( lineupid char(100),           stationid char(12),   "
01268         "  channel char(5),              channelMinor char(3) )";
01269 
01270 
01271     dd_tables["dd_v_station"] =
01272         "( stationid char(12),           callsign char(10),     "
01273         "  stationname varchar(40),      affiliate varchar(25), "
01274         "  fccchannelnumber char(15),    channel char(5),       "
01275         "  channelMinor char(3) )";
01276 
01277     dd_tables["dd_schedule"] =
01278         "( programid char(40),           stationid char(12), "
01279         "  scheduletime datetime,        duration time,      "
01280         "  isrepeat bool,                stereo bool,        "
01281         "  dolby bool, "
01282         "  subtitled bool,               hdtv bool,          "
01283         "  closecaptioned bool,          tvrating char(5),   "
01284         "  partnumber int,               parttotal int,      "
01285         "  endtime datetime,             isnew bool,         "
01286         "INDEX progidx (programid) )";
01287 
01288     dd_tables["dd_program"] =
01289         "( programid char(40) NOT NULL,  seriesid char(12),     "
01290         "  title varchar(120),           subtitle varchar(150), "
01291         "  description text,             mpaarating char(5),    "
01292         "  starrating char(5),           runtime time,          "
01293         "  year char(4),                 showtype char(30),     "
01294         "  category_type char(64),       colorcode char(20),    "
01295         "  originalairdate date,         syndicatedepisodenumber char(20), "
01296         "  stars float unsigned, "
01297         "PRIMARY KEY (programid))";
01298 
01299     dd_tables["dd_v_program"] =
01300         "( chanid int unsigned NOT NULL, starttime datetime NOT NULL, "
01301         "  endtime datetime,             title varchar(128),          "
01302         "  subtitle varchar(128),        description text,            "
01303         "  category varchar(64),         category_type varchar(64),   "
01304         "  airdate year,                 stars float unsigned,        "
01305         "  previouslyshown tinyint,      isrepeat bool,               "
01306         "  stereo bool,                  dolby bool,                  "
01307         "  subtitled bool,              "
01308         "  hdtv bool,                    closecaptioned bool,         "
01309         "  partnumber int,               parttotal int,               "
01310         "  seriesid char(12),            originalairdate date,        "
01311         "  showtype varchar(30),         colorcode varchar(20),       "
01312         "  syndicatedepisodenumber varchar(20), programid char(40),   "
01313         "  tvrating char(5),             mpaarating char(5),          "
01314         "INDEX progidx (programid))";
01315 
01316     dd_tables["dd_productioncrew"] =
01317         "( programid char(40),           role char(30),    "
01318         "  givenname char(20),           surname char(20), "
01319         "  fullname char(41), "
01320         "INDEX progidx (programid), "
01321         "INDEX nameidx (fullname))";
01322 
01323     dd_tables["dd_genre"] =
01324         "( programid char(40) NOT NULL,  class char(30), "
01325         "  relevance char(1), "
01326         "INDEX progidx (programid))";
01327 
01328     QMap<QString,QString>::const_iterator it;
01329     for (it = dd_tables.begin(); it != dd_tables.end(); ++it)
01330         CreateATempTable(it.key(), *it);
01331 }
01332 
01333 bool DataDirectProcessor::GrabLoginCookiesAndLineups(bool parse_lineups)
01334 {
01335     LOG(VB_GENERAL, LOG_INFO, LOC + "Grabbing login cookies and lineups");
01336 
01337     PostList list;
01338     list.push_back(PostItem("username", GetUserID()));
01339     list.push_back(PostItem("password", GetPassword()));
01340     list.push_back(PostItem("action",   "Login"));
01341 
01342     QString labsURL   = m_providers[m_listingsProvider].webURL;
01343     QString loginPage = m_providers[m_listingsProvider].loginPage;
01344 
01345     bool ok;
01346     QString resultFilename = GetResultFilename(ok);
01347     if (!ok)
01348     {
01349         LOG(VB_GENERAL, LOG_ERR, LOC + "GrabLoginCookiesAndLineups: "
01350                 "Creating temp result file");
01351         return false;
01352     }
01353     QString cookieFilename = GetCookieFilename(ok);
01354     if (!ok)
01355     {
01356         LOG(VB_GENERAL, LOG_ERR, LOC + "GrabLoginCookiesAndLineups: "
01357                 "Creating temp cookie file");
01358         return false;
01359     }
01360 
01361     ok = Post(labsURL + loginPage, list, resultFilename, "",
01362               cookieFilename);
01363 
01364     bool got_cookie = QFileInfo(cookieFilename).size() > 100;
01365 
01366     ok &= got_cookie && (!parse_lineups || ParseLineups(resultFilename));
01367     if (ok)
01368         m_cookieFileDT = QDateTime::currentDateTime();
01369 
01370     return ok;
01371 }
01372 
01373 bool DataDirectProcessor::GrabLineupForModify(const QString &lineupid)
01374 {
01375     LOG(VB_GENERAL, LOG_INFO, LOC +
01376         QString("Grabbing lineup %1 for modification").arg(lineupid));
01377 
01378     RawLineupMap::const_iterator it = m_rawLineups.find(lineupid);
01379     if (it == m_rawLineups.end())
01380         return false;
01381 
01382     PostList list;
01383     list.push_back(PostItem("udl_id",    GetRawUDLID(lineupid)));
01384     list.push_back(PostItem("zipcode",   GetRawZipCode(lineupid)));
01385     list.push_back(PostItem("lineup_id", lineupid));
01386     list.push_back(PostItem("submit",    "Modify"));
01387 
01388     bool ok;
01389     QString resultFilename = GetResultFilename(ok);
01390     if (!ok)
01391     {
01392         LOG(VB_GENERAL, LOG_ERR, LOC + "GrabLoginCookiesAndLineups: "
01393                 "Creating temp result file");
01394         return false;
01395     }
01396     QString cookieFilename = GetCookieFilename(ok);
01397     if (!ok)
01398     {
01399         LOG(VB_GENERAL, LOG_ERR, LOC + "GrabLoginCookiesAndLineups: "
01400                 "Creating temp cookie file");
01401         return false;
01402     }
01403 
01404     QString labsURL = m_providers[m_listingsProvider].webURL;
01405     ok = Post(labsURL + (*it).get_action, list, resultFilename,
01406                    cookieFilename, "");
01407 
01408     return ok && ParseLineup(lineupid, resultFilename);
01409 }
01410 
01411 void DataDirectProcessor::SetAll(const QString &lineupid, bool val)
01412 {
01413     LOG(VB_GENERAL, LOG_INFO, LOC + QString("%1 all channels in lineup %2")
01414             .arg((val) ? "Selecting" : "Deselecting").arg(lineupid));
01415 
01416     RawLineupMap::iterator lit = m_rawLineups.find(lineupid);
01417     if (lit == m_rawLineups.end())
01418         return;
01419 
01420     RawLineupChannels &ch = (*lit).channels;
01421     for (RawLineupChannels::iterator it = ch.begin(); it != ch.end(); ++it)
01422         (*it).chk_checked = val;
01423 }
01424 
01425 static QString get_cache_filename(const QString &lineupid)
01426 {
01427     return QString("/tmp/.mythtv_cached_lineup_") + lineupid;
01428 }
01429 
01430 QDateTime DataDirectProcessor::GetLineupCacheAge(const QString &lineupid) const
01431 {
01432     QDateTime cache_dt(QDate(1971, 1, 1));
01433     QFile lfile(get_cache_filename(lineupid));
01434     if (!lfile.exists())
01435     {
01436         LOG(VB_GENERAL, LOG_ERR, LOC + "GrabLineupCacheAge("+lineupid+
01437             ") failed -- " +
01438             QString("file '%1' doesn't exist")
01439                 .arg(get_cache_filename(lineupid)));
01440         return cache_dt;
01441     }
01442     if (lfile.size() < 8)
01443     {
01444         LOG(VB_GENERAL, LOG_ERR, LOC + "GrabLineupCacheAge("+lineupid+
01445             ") failed -- " +
01446             QString("file '%1' size %2 too small")
01447                 .arg(get_cache_filename(lineupid)).arg(lfile.size()));
01448         return cache_dt;
01449     }
01450     if (!lfile.open(QIODevice::ReadOnly))
01451     {
01452         LOG(VB_GENERAL, LOG_ERR, LOC + "GrabLineupCacheAge("+lineupid+
01453             ") failed -- " +
01454             QString("cannot open file '%1'")
01455                 .arg(get_cache_filename(lineupid)));
01456         return cache_dt;
01457     }
01458 
01459     QString tmp;
01460     QTextStream io(&lfile);
01461     io >> tmp;
01462     cache_dt = QDateTime::fromString(tmp, Qt::ISODate);
01463 
01464     LOG(VB_GENERAL, LOG_INFO, LOC + "GrabLineupCacheAge("+lineupid+") -> " +
01465             cache_dt.toString(Qt::ISODate));
01466 
01467     return cache_dt;
01468 }
01469 
01470 bool DataDirectProcessor::GrabLineupsFromCache(const QString &lineupid)
01471 {
01472     QFile lfile(get_cache_filename(lineupid));
01473     if (!lfile.exists() || (lfile.size() < 8) ||
01474         !lfile.open(QIODevice::ReadOnly))
01475     {
01476         LOG(VB_GENERAL, LOG_ERR, LOC + "GrabLineupFromCache("+lineupid+
01477             ") -- failed");
01478         return false;
01479     }
01480 
01481     QString tmp;
01482     uint size;
01483     QTextStream io(&lfile);
01484     io >> tmp; // read in date
01485     io >> size; // read in number of channels mapped
01486 
01487     for (uint i = 0; i < 14; i++)
01488         io.readLine(); // read extra lines
01489 
01490     DDLineupChannels &channels = m_lineupmaps[lineupid];
01491     channels.clear();
01492 
01493     for (uint i = 0; i < size; i++)
01494     {
01495         io.readLine(); // read "start record" string
01496 
01497         DataDirectLineupMap chan;
01498         chan.lineupid     = lineupid;
01499         chan.stationid    = io.readLine();
01500         chan.channel      = io.readLine();
01501         chan.channelMinor = io.readLine();
01502 
01503         chan.mapFrom = QDate();
01504         tmp = io.readLine();
01505         if (!tmp.isEmpty())
01506             chan.mapFrom.fromString(tmp, Qt::ISODate);
01507 
01508         chan.mapTo = QDate();
01509         tmp = io.readLine();
01510         if (!tmp.isEmpty())
01511             chan.mapTo.fromString(tmp, Qt::ISODate);
01512 
01513         channels.push_back(chan);
01514 
01515         DDStation station;
01516         station.stationid   = chan.stationid;
01517         station.callsign    = io.readLine();
01518         station.stationname = io.readLine();
01519         station.affiliate   = io.readLine();
01520         station.fccchannelnumber = io.readLine();
01521         tmp = io.readLine(); // read "end record" string
01522 
01523         m_stations[station.stationid] = station;
01524     }
01525 
01526     LOG(VB_GENERAL, LOG_INFO, LOC + "GrabLineupFromCache("+lineupid+
01527         ") -- success");
01528 
01529     return true;
01530 }
01531 
01532 bool DataDirectProcessor::SaveLineupToCache(const QString &lineupid) const
01533 {
01534     QString fn = get_cache_filename(lineupid);
01535     QByteArray fna = fn.toAscii();
01536     QFile lfile(fna.constData());
01537     if (!lfile.open(QIODevice::WriteOnly))
01538     {
01539         LOG(VB_GENERAL, LOG_ERR, LOC + "SaveLineupToCache("+lineupid+
01540             ") -- failed");
01541         return false;
01542     }
01543 
01544     QTextStream io(&lfile);
01545     io << QDateTime::currentDateTime().toString(Qt::ISODate) << endl;
01546 
01547     const DDLineupChannels channels = GetDDLineup(lineupid);
01548     io << channels.size() << endl;
01549 
01550     io << endl;
01551     io << "# start record"       << endl;
01552     io << "#   stationid"        << endl;
01553     io << "#   channel"          << endl;
01554     io << "#   channelMinor"     << endl;
01555     io << "#   mapped from date" << endl;
01556     io << "#   mapped to date"   << endl;
01557     io << "#   callsign"         << endl;
01558     io << "#   stationname"      << endl;
01559     io << "#   affiliate"        << endl;
01560     io << "#   fccchannelnumber" << endl;
01561     io << "# end record"         << endl;
01562     io << endl;
01563 
01564     DDLineupChannels::const_iterator it;
01565     for (it = channels.begin(); it != channels.end(); ++it)
01566     {
01567         io << "# start record"    << endl;
01568         io << (*it).stationid     << endl;
01569         io << (*it).channel       << endl;
01570         io << (*it).channelMinor  << endl;
01571         io << (*it).mapFrom.toString(Qt::ISODate) << endl;
01572         io << (*it).mapTo.toString(Qt::ISODate)   << endl;
01573 
01574         DDStation station = GetDDStation((*it).stationid);
01575         io << station.callsign    << endl;
01576         io << station.stationname << endl;
01577         io << station.affiliate   << endl;
01578         io << station.fccchannelnumber << endl;
01579         io << "# end record"      << endl;
01580     }
01581     io << flush;
01582 
01583     LOG(VB_GENERAL, LOG_INFO, LOC + "SaveLineupToCache("+lineupid+
01584         ") -- success");
01585 
01586     makeFileAccessible(fna.constData()); // Let anybody update it
01587 
01588     return true;
01589 }
01590 
01591 bool DataDirectProcessor::GrabFullLineup(const QString &lineupid,
01592                                          bool restore, bool onlyGrabSelected,
01593                                          uint cache_age_allowed_in_seconds)
01594 {
01595     if (cache_age_allowed_in_seconds)
01596     {
01597         QDateTime exp_time = GetLineupCacheAge(lineupid)
01598             .addSecs(cache_age_allowed_in_seconds);
01599         bool valid = exp_time > QDateTime::currentDateTime();
01600         if (valid && GrabLineupsFromCache(lineupid))
01601             return true;
01602     }
01603 
01604     bool ok = GrabLoginCookiesAndLineups();
01605     if (!ok)
01606         return false;
01607 
01608     ok = GrabLineupForModify(lineupid);
01609     if (!ok)
01610         return false;
01611 
01612     RawLineupMap::iterator lit = m_rawLineups.find(lineupid);
01613     if (lit == m_rawLineups.end())
01614         return false;
01615 
01616     const RawLineupChannels orig_channels = (*lit).channels;
01617 
01618     if (!onlyGrabSelected)
01619     {
01620         SetAll(lineupid, true);
01621         if (!SaveLineupChanges(lineupid))
01622             return false;
01623     }
01624 
01625     ok = GrabLineupsOnly();
01626 
01627     if (ok)
01628         SaveLineupToCache(lineupid);
01629 
01630     (*lit).channels = orig_channels;
01631     if (restore && !onlyGrabSelected)
01632         ok &= SaveLineupChanges(lineupid);
01633 
01634     return ok;
01635 }
01636 
01637 bool DataDirectProcessor::SaveLineup(const QString &lineupid,
01638                                      const QMap<QString,bool> &xmltvids)
01639 {
01640     QMap<QString,bool> callsigns;
01641     RawLineupMap::iterator lit = m_rawLineups.find(lineupid);
01642     if (lit == m_rawLineups.end())
01643         return false;
01644 
01645     // Grab login cookies if they are more than 5 minutes old
01646     if ((!m_cookieFileDT.isValid() ||
01647          m_cookieFileDT.addSecs(5*60) < QDateTime::currentDateTime()) &&
01648         !GrabLoginCookiesAndLineups(false))
01649     {
01650         return false;
01651     }
01652 
01653     // Get callsigns based on xmltv ids (aka stationid)
01654     DDLineupMap::const_iterator ddit = m_lineupmaps.find(lineupid);
01655     DDLineupChannels::const_iterator it;
01656     for (it = (*ddit).begin(); it != (*ddit).end(); ++it)
01657     {
01658         if (xmltvids.find((*it).stationid) != xmltvids.end())
01659             callsigns[GetDDStation((*it).stationid).callsign] = true;
01660     }
01661 
01662     // Set checked mark based on whether the channel is mapped
01663     RawLineupChannels &ch = (*lit).channels;
01664     RawLineupChannels::iterator cit;
01665     for (cit = ch.begin(); cit != ch.end(); ++cit)
01666     {
01667         bool chk = callsigns.find((*cit).lbl_callsign) != callsigns.end();
01668         (*cit).chk_checked = chk;
01669     }
01670 
01671     // Save these changes
01672     return SaveLineupChanges(lineupid);
01673 }
01674 
01675 bool DataDirectProcessor::SaveLineupChanges(const QString &lineupid)
01676 {
01677     RawLineupMap::const_iterator lit = m_rawLineups.find(lineupid);
01678     if (lit == m_rawLineups.end())
01679         return false;
01680 
01681     const RawLineup &lineup = *lit;
01682     const RawLineupChannels &ch = lineup.channels;
01683     RawLineupChannels::const_iterator it;
01684 
01685     PostList list;
01686     for (it = ch.begin(); it != ch.end(); ++it)
01687     {
01688         if ((*it).chk_checked)
01689             list.push_back(PostItem((*it).chk_name, (*it).chk_value));
01690     }
01691     list.push_back(PostItem("action", "Update"));
01692 
01693     LOG(VB_GENERAL, LOG_INFO, LOC + QString("Saving lineup %1 with %2 channels")
01694             .arg(lineupid).arg(list.size() - 1));
01695 
01696     bool ok;
01697     QString cookieFilename = GetCookieFilename(ok);
01698     if (!ok)
01699     {
01700         LOG(VB_GENERAL, LOG_ERR, LOC + "GrabLoginCookiesAndLineups: "
01701                 "Creating temp cookie file");
01702         return false;
01703     }
01704 
01705     QString labsURL = m_providers[m_listingsProvider].webURL;
01706     return Post(labsURL + lineup.set_action, list, "",
01707                 cookieFilename, "");
01708 }
01709 
01710 bool DataDirectProcessor::UpdateListings(uint sourceid)
01711 {
01712     MSqlQuery query(MSqlQuery::DDCon());
01713     query.prepare(
01714         "SELECT xmltvid "
01715         "FROM channel "
01716         "WHERE sourceid = :SOURCEID");
01717     query.bindValue(":SOURCEID", sourceid);
01718 
01719     if (!query.exec() || !query.isActive())
01720     {
01721         MythDB::DBError("Selecting existing channels", query);
01722         return false;
01723     }
01724 
01725     QString a, b, c, lineupid;
01726     if (!SourceUtil::GetListingsLoginData(sourceid, a, b, c, lineupid))
01727         return false;
01728 
01729     QMap<QString,bool> xmltvids;
01730     while (query.next())
01731     {
01732         if (!query.value(0).toString().isEmpty())
01733             xmltvids[query.value(0).toString()] = true;
01734     }
01735 
01736     LOG(VB_GENERAL, LOG_INFO, LOC + "Saving updated DataDirect listing");
01737     bool ok = SaveLineup(lineupid, xmltvids);
01738 
01739     if (!ok)
01740         LOG(VB_GENERAL, LOG_ERR, LOC + "Failed to update DataDirect listings.");
01741 
01742     return ok;
01743 }
01744 
01745 QDateTime  DataDirectProcessor::GetDDProgramsStartAt(bool localtime) const
01746 {
01747     if (localtime)
01748         return MythUTCToLocal(m_actualListingsFrom);
01749     return m_actualListingsFrom;
01750 }
01751 
01752 QDateTime  DataDirectProcessor::GetDDProgramsEndAt(bool localtime) const
01753 {
01754     if (localtime)
01755         return MythUTCToLocal(m_actualListingsTo);
01756     return m_actualListingsTo;
01757 }
01758 
01759 QString DataDirectProcessor::GetRawUDLID(const QString &lineupid) const
01760 {
01761     RawLineupMap::const_iterator it = m_rawLineups.find(lineupid);
01762     if (it == m_rawLineups.end())
01763         return QString::null;
01764     return (*it).udl_id;
01765 }
01766 
01767 QString DataDirectProcessor::GetRawZipCode(const QString &lineupid) const
01768 {
01769     RawLineupMap::const_iterator it = m_rawLineups.find(lineupid);
01770     if (it == m_rawLineups.end())
01771         return QString::null;
01772     return (*it).zipcode;
01773 }
01774 
01775 RawLineup DataDirectProcessor::GetRawLineup(const QString &lineupid) const
01776 {
01777     RawLineup tmp;
01778     RawLineupMap::const_iterator it = m_rawLineups.find(lineupid);
01779     if (it == m_rawLineups.end())
01780         return tmp;
01781     return (*it);
01782 }
01783 
01784 void DataDirectProcessor::CreateTemp(
01785     const QString &templatefilename,
01786     const QString &errmsg,
01787     bool           directory,
01788     QString       &filename,
01789     bool          &ok) const
01790 {
01791     QString tmp = createTempFile(templatefilename, directory);
01792     if (templatefilename == tmp)
01793     {
01794         m_fatalErrors.push_back(errmsg);
01795         ok = false;
01796     }
01797     else
01798     {
01799         filename = tmp;
01800         ok = true;
01801     }
01802 }
01803 
01804 QString DataDirectProcessor::CreateTempDirectory(bool *pok) const
01805 {
01806     bool ok;
01807     pok = (pok) ? pok : &ok;
01808     if (m_tmpDir == "/tmp")
01809     {
01810         CreateTemp("/tmp/mythtv_ddp_XXXXXX",
01811                    "Failed to create temp directory",
01812                    true, m_tmpDir, *pok);
01813     }
01814     return m_tmpDir;
01815 }
01816 
01817 
01818 QString DataDirectProcessor::GetResultFilename(bool &ok) const
01819 {
01820     ok = true;
01821     if (m_tmpResultFile.isEmpty())
01822     {
01823         CreateTemp(m_tmpDir + "/mythtv_result_XXXXXX",
01824                    "Failed to create temp result file",
01825                    false, m_tmpResultFile, ok);
01826     }
01827     return m_tmpResultFile;
01828 }
01829 
01830 QString DataDirectProcessor::GetCookieFilename(bool &ok) const
01831 {
01832     ok = true;
01833     if (m_cookieFile.isEmpty())
01834     {
01835         CreateTemp(m_tmpDir + "/mythtv_cookies_XXXXXX",
01836                    "Failed to create temp cookie file",
01837                    false, m_cookieFile, ok);
01838     }
01839     return m_cookieFile;
01840 }
01841 
01842 void DataDirectProcessor::SetUserID(const QString &uid)
01843 {
01844     m_userid = uid;
01845     m_userid.detach();
01846 }
01847 
01848 void DataDirectProcessor::SetPassword(const QString &pwd)
01849 {
01850     m_password = pwd;
01851     m_password.detach();
01852 }
01853 
01854 void DataDirectProcessor::SetInputFile(const QString &file)
01855 {
01856     m_inputFilename = file;
01857     m_inputFilename.detach();
01858 }
01859 
01860 bool DataDirectProcessor::Post(QString url, const PostList &list,
01861                                QString documentFile,
01862                                QString inCookieFile, QString outCookieFile)
01863 {
01864     MythDownloadManager *manager = GetMythDownloadManager();
01865 
01866     if (!inCookieFile.isEmpty())
01867         manager->loadCookieJar(inCookieFile);
01868 
01869     QByteArray postdata;
01870     for (uint i = 0; i < list.size(); i++)
01871     {
01872         postdata += ((i) ? "&" : "") + list[i].key + "=";
01873         postdata += html_escape(list[i].value);
01874     }
01875 
01876     if (!manager->post(url, &postdata))
01877         return false;
01878 
01879     if (!outCookieFile.isEmpty())
01880         manager->saveCookieJar(outCookieFile);
01881 
01882     if (documentFile.isEmpty())
01883         return true;
01884 
01885     QFile file(documentFile);
01886     file.open(QIODevice::WriteOnly);
01887     file.write(postdata);
01888     file.close();
01889 
01890     QFileInfo fi(documentFile);
01891     return fi.size();
01892 }
01893 
01894 bool DataDirectProcessor::ParseLineups(const QString &documentFile)
01895 {
01896     QFile file(documentFile);
01897     if (!file.open(QIODevice::ReadOnly))
01898     {
01899         LOG(VB_GENERAL, LOG_ERR, LOC + QString("Failed to open '%1'")
01900                 .arg(documentFile));
01901         return false;
01902     }
01903 
01904     QTextStream stream(&file);
01905     bool in_form = false;
01906     QString get_action = QString::null;
01907     QMap<QString,QString> name_value;
01908 
01909     m_rawLineups.clear();
01910 
01911     while (!stream.atEnd())
01912     {
01913         QString line = stream.readLine();
01914         QString llow = line.toLower();
01915         int frm = llow.indexOf("<form");
01916         if (frm >= 0)
01917         {
01918             in_form = true;
01919             get_action = get_setting(line.mid(frm + 5), "action");
01920             name_value.clear();
01921 #if 0
01922             LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("action: %1").arg(action));
01923 #endif
01924         }
01925 
01926         if (!in_form)
01927             continue;
01928 
01929         int inp = llow.indexOf("<input");
01930         if (inp >= 0)
01931         {
01932             QString input_line = line.mid(inp + 6);
01933 #if 0
01934             LOG(VB_GENERAL, LOG_DEBUG, LOC +
01935                 QString("input: %1").arg(input_line));
01936 #endif
01937             QString name  = get_setting(input_line, "name");
01938             QString value = get_setting(input_line, "value");
01939 #if 0
01940             LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("name: %1").arg(name));
01941             LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("value: %1").arg(value));
01942 #endif
01943             if (!name.isEmpty() && !value.isEmpty())
01944                 name_value[name] = value;
01945         }
01946 
01947         if (llow.contains("</form>"))
01948         {
01949             in_form = false;
01950             if (!get_action.isEmpty() &&
01951                 !name_value["udl_id"].isEmpty() &&
01952                 !name_value["zipcode"].isEmpty() &&
01953                 !name_value["lineup_id"].isEmpty())
01954             {
01955                 RawLineup item(get_action, name_value["udl_id"],
01956                                name_value["zipcode"]);
01957 
01958                 m_rawLineups[name_value["lineup_id"]] = item;
01959 #if 0
01960                 LOG(VB_GENERAL, LOG_DEBUG, LOC +
01961                     QString("<%1>  \t--> <%2,%3,%4>")
01962                         .arg(name_value["lineup_id"])
01963                         .arg(item.udl_id).arg(item.zipcode)
01964                         .arg(item.get_action));
01965 #endif
01966             }
01967         }
01968     }
01969     return true;
01970 }
01971 
01972 bool DataDirectProcessor::ParseLineup(const QString &lineupid,
01973                                       const QString &documentFile)
01974 {
01975     QFile file(documentFile);
01976     if (!file.open(QIODevice::ReadOnly))
01977     {
01978         LOG(VB_GENERAL, LOG_ERR, LOC + QString("Failed to open '%1'")
01979                 .arg(documentFile));
01980 
01981         return false;
01982     }
01983 
01984     QTextStream stream(&file);
01985     bool in_form = false;
01986     int in_label = 0;
01987     QMap<QString,QString> settings;
01988 
01989     RawLineup &lineup = m_rawLineups[lineupid];
01990     RawLineupChannels &ch = lineup.channels;
01991 
01992     while (!stream.atEnd())
01993     {
01994         QString line = stream.readLine();
01995         QString llow = line.toLower();
01996         int frm = llow.indexOf("<form");
01997         if (frm >= 0)
01998         {
01999             in_form = true;
02000             lineup.set_action = get_setting(line.mid(frm + 5), "action");
02001 #if 0
02002             LOG(VB_GENERAL, LOG_DEBUG, LOC + "set_action: " +
02003                 lineup.set_action);
02004 #endif
02005         }
02006 
02007         if (!in_form)
02008             continue;
02009 
02010         int inp = llow.indexOf("<input");
02011         if (inp >= 0)
02012         {
02013             QString in_line = line.mid(inp + 6);
02014             settings.clear();
02015             settings["chk_name"]    = get_setting(in_line, "name");
02016             settings["chk_id"]      = get_setting(in_line, "id");
02017             settings["chk_value"]   = get_setting(in_line, "value");
02018             settings["chk_checked"] = has_setting(in_line, "checked")?"1":"0";
02019         }
02020 
02021         int lbl = llow.indexOf("<label");
02022         if (lbl >= 0)
02023         {
02024             QString lbl_line = line.mid(inp + 6);
02025             QString name = get_setting(lbl_line, "for");
02026             in_label = (name == settings["chk_name"]) ? 1 : 0;
02027         }
02028 
02029         if (in_label)
02030         {
02031             int start = (lbl >= 0) ? lbl + 6 : 0;
02032             int beg = llow.indexOf("<td>", start), end = -1;
02033             if (beg)
02034                 end = llow.indexOf("</td>", beg + 4);
02035 
02036             if (end >= 0)
02037             {
02038                 QString key = (in_label == 1) ? "lbl_ch" : "lbl_callsign";
02039                 QString val = line.mid(beg + 4, end - beg - 4);
02040                 settings[key] = val.replace("&nbsp;", "", Qt::CaseInsensitive);
02041                 in_label++;
02042             }
02043         }
02044 
02045         in_label = (llow.indexOf("</label") >= 0) ? 0 : in_label;
02046 
02047         if (!in_label &&
02048             !settings["chk_name"].isEmpty() &&
02049             !settings["chk_id"].isEmpty() &&
02050             !settings["chk_value"].isEmpty() &&
02051             !settings["chk_checked"].isEmpty() &&
02052             !settings["lbl_ch"].isEmpty() &&
02053             !settings["lbl_callsign"].isEmpty())
02054         {
02055             RawLineupChannel chan(
02056                 settings["chk_name"],  settings["chk_id"],
02057                 settings["chk_value"], settings["chk_checked"] == "1",
02058                 settings["lbl_ch"],    settings["lbl_callsign"]);
02059 
02060 #if 0
02061             LOG(VB_GENERAL, LOG_DEBUG, LOC +
02062                 QString("name: %1  id: %2  value: %3  "
02063                         "checked: %4  ch: %5  call: %6")
02064                     .arg(settings["chk_name"]).arg(settings["chk_id"])
02065                     .arg(settings["chk_value"]).arg(settings["chk_checked"])
02066                     .arg(settings["lbl_ch"],4).arg(settings["lbl_callsign"]));
02067 #endif
02068 
02069             ch.push_back(chan);
02070             settings.clear();
02071         }
02072 
02073         if (llow.contains("</form>"))
02074         {
02075             in_form = false;
02076         }
02077     }
02078     return true;
02079 }
02080 
02081 static QString html_escape(QString str)
02082 {
02083     QString new_str = "";
02084     for (int i = 0; i < str.length(); i++)
02085     {
02086         if (str[i].isLetterOrNumber())
02087             new_str += str[i];
02088         else
02089         {
02090             new_str += QString("%%1").arg((int)str[1].toLatin1(), 0, 16);
02091         }
02092     }
02093 
02094     return new_str;
02095 }
02096 
02097 static QString get_setting(QString line, QString key)
02098 {
02099     QString llow = line.toLower();
02100     QString kfind = key + "=\"";
02101     int beg = llow.indexOf(kfind), end = -1;
02102 
02103     if (beg >= 0)
02104     {
02105         end = llow.indexOf("\"", beg + kfind.length());
02106         return line.mid(beg + kfind.length(), end - beg - kfind.length());
02107     }
02108 
02109     kfind = key + "=";
02110     beg = llow.indexOf(kfind);
02111     if (beg < 0)
02112         return QString::null;
02113 
02114     int i = beg + kfind.length();
02115     while (i < line.length() && !line[i].isSpace() && line[i] != '>')
02116         i++;
02117 
02118     if (i < line.length() && (line[i].isSpace() || line[i] == '>'))
02119         return line.mid(beg + kfind.length(), i - beg - kfind.length());
02120 
02121     return QString::null;
02122 }
02123 
02124 static bool has_setting(QString line, QString key)
02125 {
02126     return (line.toLower().indexOf(key) >= 0);
02127 }
02128 
02129 static void get_atsc_stuff(QString channum, int sourceid, int freqid,
02130                            int &major, int &minor, long long &freq)
02131 {
02132     major = freqid;
02133     minor = 0;
02134 
02135     int chansep = channum.indexOf(QRegExp("\\D"));
02136     if (chansep < 0)
02137         return;
02138 
02139     major = channum.left(chansep).toInt();
02140     minor = channum.right(channum.length() - (chansep + 1)).toInt();
02141 
02142     freq = get_center_frequency("atsc", "vsb8", "us", freqid);
02143 }
02144 
02145 static QString process_dd_station(
02146     uint sourceid, QString chan_major, QString chan_minor,
02147     QString &tvformat, uint &freqid)
02148 {
02149     QString channum = chan_major;
02150     bool ok;
02151     uint minor = chan_minor.toUInt(&ok);
02152 
02153     tvformat = "Default";
02154 
02155     if (minor && ok)
02156     {
02157         tvformat = "atsc";
02158         channum += SourceUtil::GetChannelSeparator(sourceid) + chan_minor;
02159     }
02160     else if (!freqid && (get_lineup_type(sourceid) == "LocalBroadcast"))
02161         freqid = chan_major.toInt();
02162     else
02163         freqid = channum.toInt();
02164 
02165     return channum;
02166 }
02167 
02168 static uint update_channel_basic(uint    sourceid,   bool    insert,
02169                                  QString xmltvid,    QString callsign,
02170                                  QString name,       uint    freqid,
02171                                  QString chan_major, QString chan_minor)
02172 {
02173     callsign = (callsign.isEmpty()) ? name : callsign;
02174 
02175     QString tvformat;
02176     QString channum = process_dd_station(
02177         sourceid, chan_major, chan_minor, tvformat, freqid);
02178 
02179     // First check if channel already in DB, but without xmltvid
02180     MSqlQuery query(MSqlQuery::DDCon());
02181     query.prepare("SELECT chanid, callsign, name "
02182                   "FROM channel "
02183                   "WHERE sourceid = :SOURCEID AND "
02184                   "      ( xmltvid = '0' OR xmltvid = '') AND "
02185                   "      ( channum = :CHANNUM OR "
02186                   "        ( freqid  = :FREQID AND "
02187                   "          freqid != '0'     AND "
02188                   "          freqid != ''      AND "
02189                   "          atsc_minor_chan = '0') OR "
02190                   "        ( atsc_major_chan = :MAJORCHAN AND "
02191                   "          atsc_minor_chan = :MINORCHAN ) )");
02192     query.bindValue(":SOURCEID",  sourceid);
02193     query.bindValue(":CHANNUM",   channum);
02194     query.bindValue(":FREQID",    freqid);
02195     query.bindValue(":MAJORCHAN", chan_major.toUInt());
02196     query.bindValue(":MINORCHAN", chan_minor.toUInt());
02197 
02198     if (!query.exec() || !query.isActive())
02199     {
02200         MythDB::DBError("Getting chanid of existing channel", query);
02201         return 0; // go on to next channel without xmltv
02202     }
02203 
02204     if (query.next())
02205     {
02206         // The channel already exists in DB, at least once,
02207         // so set the xmltvid..
02208         MSqlQuery chan_update_q(MSqlQuery::DDCon());
02209         chan_update_q.prepare(
02210             "UPDATE channel "
02211             "SET xmltvid = :XMLTVID, name = :NAME, callsign = :CALLSIGN "
02212             "WHERE chanid = :CHANID AND sourceid = :SOURCEID");
02213 
02214         uint i = 0;
02215         do
02216         {
02217             uint chanid = query.value(0).toInt();
02218 
02219             QString new_callsign = query.value(1).toString();
02220             new_callsign =
02221                 (new_callsign.indexOf(ChannelUtil::GetUnknownCallsign()) == 0) ?
02222                 callsign : new_callsign;
02223 
02224             QString new_name = query.value(2).toString();
02225             new_name = (new_name.isEmpty()) ? name         : new_name;
02226             new_name = (new_name.isEmpty()) ? new_callsign : new_name;
02227 
02228             chan_update_q.bindValue(":CHANID",   chanid);
02229             chan_update_q.bindValue(":NAME",     new_name);
02230             chan_update_q.bindValue(":CALLSIGN", new_callsign);
02231             chan_update_q.bindValue(":XMLTVID",  xmltvid);
02232             chan_update_q.bindValue(":SOURCEID", sourceid);
02233 
02234 #if 0
02235             LOG(VB_GENERAL, LOG_INFO, LOC +
02236                 QString("Updating channel %1: '%2' (%3).")
02237                     .arg(chanid).arg(name).arg(callsign));
02238 #endif
02239 
02240             if (!chan_update_q.exec() || !chan_update_q.isActive())
02241             {
02242                 MythDB::DBError(
02243                     "Updating XMLTVID of existing channel", chan_update_q);
02244                 continue; // go on to next instance of this channel
02245             }
02246             i++;
02247         }
02248         while (query.next());
02249 
02250         return i; // go on to next channel without xmltv
02251     }
02252 
02253     if (!insert)
02254         return 0; // go on to next channel without xmltv
02255 
02256     // The channel doesn't exist in the DB, insert it...
02257     int mplexid = -1, majorC, minorC, chanid = 0;
02258     long long freq = -1;
02259     get_atsc_stuff(channum, sourceid, freqid,
02260                    majorC, minorC, freq);
02261 
02262     if (minorC > 0 && freq >= 0)
02263         mplexid = ChannelUtil::CreateMultiplex(sourceid, "atsc", freq, "8vsb");
02264 
02265     if ((mplexid > 0) || (minorC == 0))
02266         chanid = ChannelUtil::CreateChanID(sourceid, channum);
02267 
02268     LOG(VB_GENERAL, LOG_INFO, LOC + QString("Adding channel %1 '%2' (%3).")
02269             .arg(channum).arg(name).arg(callsign));
02270 
02271     if (chanid > 0)
02272     {
02273         QString icon   = "";
02274         int  serviceid = 0;
02275         bool oag       = false; // use on air guide
02276         bool hidden    = false;
02277         bool hidden_in_guide = false;
02278         QString freq_id= QString::number(freqid);
02279 
02280         ChannelUtil::CreateChannel(
02281             mplexid,   sourceid,  chanid,
02282             callsign,  name,      channum,
02283             serviceid, majorC,    minorC,
02284             oag,       hidden,    hidden_in_guide,
02285             freq_id,   icon,      tvformat,
02286             xmltvid);
02287     }
02288 
02289     return 1;
02290 }
02291 
02292 static void set_lineup_type(const QString &lineupid, const QString &type)
02293 {
02294     QMutexLocker locker(&lineup_type_lock);
02295     if (lineupid_to_srcid[lineupid])
02296         return;
02297 
02298     // get lineup to source mapping
02299     uint srcid = 0;
02300     MSqlQuery query(MSqlQuery::InitCon());
02301     query.prepare(
02302         "SELECT sourceid "
02303         "FROM videosource "
02304         "WHERE lineupid = :LINEUPID");
02305     query.bindValue(":LINEUPID", lineupid);
02306 
02307     if (!query.exec() || !query.isActive())
02308         MythDB::DBError("end_element", query);
02309     else if (query.next())
02310         srcid = query.value(0).toUInt();
02311 
02312     if (srcid)
02313     {
02314         QString tmpid = lineupid;
02315         tmpid.detach();
02316         lineupid_to_srcid[tmpid] = srcid;
02317 
02318         // set type for source
02319         QString tmptype = type;
02320         tmptype.detach();
02321         srcid_to_type[srcid] = tmptype;
02322 
02323         LOG(VB_GENERAL, LOG_INFO, LOC +
02324             QString("sourceid %1 has lineup type: %2").arg(srcid).arg(type));
02325     }
02326 }
02327 
02328 static QString get_lineup_type(uint sourceid)
02329 {
02330     QMutexLocker locker(&lineup_type_lock);
02331     QString ret = srcid_to_type[sourceid];
02332     ret.detach();
02333     return ret;
02334 }
02335 
02336 /*
02337  * This function taken from: 
02338  * http://stackoverflow.com/questions/2690328/qt-quncompress-gzip-data
02339  *
02340  * Based on zlib example code.
02341  *
02342  * Copyright (c) 2011 Ralf Engels <ralf-engels@gmx.de>
02343  * Copyright (C) 1995-2012 Jean-loup Gailly and Mark Adler
02344  *
02345  * Licensed under the terms of the ZLib license which is found at
02346  * http://zlib.net/zlib_license.html and is as follows:
02347  *
02348  *  This software is provided 'as-is', without any express or implied
02349  *  warranty.  In no event will the authors be held liable for any damages
02350  *  arising from the use of this software.
02351  *
02352  *  Permission is granted to anyone to use this software for any purpose,
02353  *  including commercial applications, and to alter it and redistribute it
02354  *  freely, subject to the following restrictions:
02355  *
02356  *  1. The origin of this software must not be misrepresented; you must not
02357  *     claim that you wrote the original software. If you use this software
02358  *     in a product, an acknowledgment in the product documentation would be
02359  *     appreciated but is not required.
02360  *  2. Altered source versions must be plainly marked as such, and must not be
02361  *     misrepresented as being the original software.
02362  *  3. This notice may not be removed or altered from any source distribution.
02363  *
02364  *  NOTE: The Zlib license is listed as being GPL-compatible
02365  *    http://www.gnu.org/licenses/license-list.html#ZLib
02366  */
02367 
02368 QByteArray gUncompress(const QByteArray &data)
02369 {
02370     if (data.size() <= 4) {
02371         LOG(VB_GENERAL, LOG_WARNING, "gUncompress: Input data is truncated");
02372         return QByteArray();
02373     }
02374 
02375     QByteArray result;
02376 
02377     int ret;
02378     z_stream strm;
02379     static const int CHUNK_SIZE = 1024;
02380     char out[CHUNK_SIZE];
02381 
02382     /* allocate inflate state */
02383     strm.zalloc = Z_NULL;
02384     strm.zfree = Z_NULL;
02385     strm.opaque = Z_NULL;
02386     strm.avail_in = data.size();
02387     strm.next_in = (Bytef*)(data.data());
02388 
02389     ret = inflateInit2(&strm, 15 +  32); // gzip decoding
02390     if (ret != Z_OK)
02391         return QByteArray();
02392 
02393     // run inflate()
02394     do {
02395         strm.avail_out = CHUNK_SIZE;
02396         strm.next_out = (Bytef*)(out);
02397 
02398         ret = inflate(&strm, Z_NO_FLUSH);
02399         Q_ASSERT(ret != Z_STREAM_ERROR);  // state not clobbered
02400 
02401         switch (ret) {
02402         case Z_NEED_DICT:
02403             ret = Z_DATA_ERROR;     // and fall through
02404         case Z_DATA_ERROR:
02405         case Z_MEM_ERROR:
02406             (void)inflateEnd(&strm);
02407             return QByteArray();
02408         }
02409 
02410         result.append(out, CHUNK_SIZE - strm.avail_out);
02411     } while (strm.avail_out == 0);
02412 
02413     // clean up and return
02414     inflateEnd(&strm);
02415     return result;
02416 }
02417 
02418 /* vim: set expandtab tabstop=4 shiftwidth=4: */