path: root/mail/freepops/patches/patch-ai

$NetBSD: patch-ai,v 1.1 2008/10/23 04:40:55 schmonz Exp $

--- src/lua/yahoo.lua.orig	2008-10-23 00:05:23.000000000 -0400
+++ src/lua/yahoo.lua
@@ -14,7 +14,7 @@
 local _DEBUG = false
 local DBG_LEN = nil -- 500
 
-PLUGIN_VERSION = "0.2.1"
+PLUGIN_VERSION = "0.2.1g"
 PLUGIN_NAME = "yahoo.com"
 PLUGIN_REQUIRE_VERSION = "0.2.0"
 PLUGIN_LICENSE = "GNU/GPL"
@@ -27,6 +27,7 @@ PLUGIN_AUTHORS_CONTACTS = 
 PLUGIN_DOMAINS = {"@yahoo.com", "@yahoo.ie", "@yahoo.it", "@yahoo.ca", "@rocketmail.com", "@yahoo.com.ar",
                   "@yahoo.co.in", "@yahoo.co.id", "@yahoo.com.tw", "@yahoo.co.uk", "@yahoo.com.cn",
                   "@yahoo.es", "@yahoo.de", "@talk21.com", "@btinternet.com", "@yahoo.com.au", "@yahoo.co.nz",
+				  "@ymail.com", "@yahoo.in"
 }
 
 PLUGIN_PARAMETERS = {
@@ -87,6 +88,18 @@ pulling messages.  Set the value to 1.]]
 Parameter is used to force the plugin to only download a maximum number of messages. ]]
 		}	
 	},
+	{name = "keepmsgstatus", description = {
+		en = [[
+Parameter is used to maintain the status of the message in the state it was before being pulling.  If the value is 1, the behavior is turned on
+and will override the markunread flag. ]]
+		}	
+	},
+	{name = "domain", description = {
+		en = [[
+Parameter is used to override the domain in the email address.  This is used so that users don't
+need to add a mapping to config.lua for a hosted hotmail account. ]]
+		}	
+	},
 }
 
 PLUGIN_DESCRIPTIONS = {
@@ -155,20 +168,28 @@ local globals = {
   -- Pattern to determine if we have no messages.  If this is found, we have messages.
   --
   strMsgListNoMsgPat = "(<tbody>)",
+  
+  -- Pattern to determine if we have no messages with 'mc'.  If found, there is no messages.
+  --
+  strMsgListNoMsgPatMC = '(modulecontainer filled nomessages)',
 
   -- Used by Stat to pull out the message ID and the size
   --
   strMsgLineLitPattern = ".*<td>[.*]{div}[.*]{h2}.*<a>.*</a>[.*]{/h2}[.*]{/div}.*</td>.*<td>.*</td>.*<td>.*</td>.*</tr>",
   strMsgLineAbsPattern = "O<O>[O]{O}[O]{O}O<X>O<O>[O]{O}[O]{O}O<O>O<O>O<O>O<O>X<O>O<O>",
-    
+
+  strMCUnreadPattern = '<tr class="([^"]+)"><td><b>[^<]+</b></td><td[^>]+><input type="checkbox" name="mid" value="([^"]+)"',    
+
   -- MSGID Pattern
   -- 
-  strMsgIDPattern = 'MsgId=([^&]*)&',
-  strMsgIDMCPattern = 'mid=([^&]*)&',
+  strMsgIDPattern = 'MsgId=([^&"]*)[&"]',
+  strMsgIDMCPattern = 'mid=([^&"]*)[&"]',
+  strMsgIDMIPattern = 'DMid=([^&"]*)[&"]',
 
   -- Pattern used by Stat to get the next page in the list of messages
   --
   strMsgListPrevPagePattern = '<a href="/([my][cm][^"]*previous=1[^"]*)">',
+  strMsgListNextPagePatternMC = '| <a href="([^"]+)"><span[^|]+| <a href="[^"]+&last=1">',
 
   -- Pattern for emptying all
   --
@@ -192,8 +213,8 @@ local globals = {
 
   -- Command URLS
   --
-  strCmdMsgList = "%s%s/ShowFolder?box=%s&Npos=%d&Nview=%s&view=%s&order=down&sort=date&reset=1&Norder=up",
-  strCmdMsgListMC = "%s%s/showFolder?fid=%s&startMid=%s&ymv=0&&sort=date&order=down&filterBySelect=%s&filterBySelect=%s&prefNumOfMid=500",
+  strCmdMsgList = "%s%s/ShowFolder?box=%s&Npos=%d&filterBySelect=%s&order=down&sort=date&reset=1&Norder=up&filterButton=Go",
+  strCmdMsgListMC = "%s%s/showFolder?fid=%s&startMid=%s&ymv=0&&sort=date&order=down&filterBySelect=%s&prefNumOfMid=500&filterButton=Go",
   strCmdMsgView = "%sya/download?box=%s&MsgId=%s&bodyPart=%s&download=1&YY=38663&y5beta=yes&y5beta=yes&order=down&sort=date&pos=0&view=a&head=b&DataName=&tnef=&Idx=0",
   strCmdMsgWebView = "%s%s/ShowLetter?box=%s&MsgId=%s",
   strCmdMsgWebViewMC = "%s%s/showMessage?fid=%s&midIndex=0&mid=%s&eps=&f=1&",
@@ -207,9 +228,9 @@ local globals = {
 
   -- Emails to list - These define the filter on the messages to grab
   --
-  strViewAll = "a",
-  strViewUnread = "u",
-  strViewFlagged = "f",
+  strViewAll = "all",
+  strViewUnread = "unread",
+  strViewFlagged = "flagged",
 
   strViewAllPat = "([Aa]ll)",
   strViewUnreadPat = "([Uu]nread)",
@@ -298,9 +319,11 @@ internalState = {
   bEmptyTrash = false,
   bEmptyBulk = false,
   msgids = {},
+  unreadMsgs = {},
   strStatCache = nil,
   statLimit = nil,
   classicType = nil,
+  bKeepMsgStatus = false,
 
   -- New Interface pieces
   --
@@ -350,7 +373,7 @@ function table_to_string(t, depth)
     )
     result = "{" .. table.concat(tmp,", ") .. "}"
   else
-    result = tostring(val)
+    result = tostring(t)
   end
   return result
 end
@@ -584,7 +607,8 @@ function loginYahoo()
 
   -- Create a browser to do the dirty work (It must be set to IE 6.0)
   --
-  internalState.browser = browser.new("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; {02ABE9F9-C33D-95EA-0C84-0B70CD0AC3F8}; .NET CLR 1.1.4322)")
+  internalState.browser = browser.new("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_2; en) AppleWebKit/525.9 (KHTML, like Gecko) Version/3.1 Safari/525.9")
+  --browser.new("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; {02ABE9F9-C33D-95EA-0C84-0B70CD0AC3F8}; .NET CLR 1.1.4322)")
 --  internalState.browser = browser.new("Mozilla/4.0 (compatible; U; en)")
 --  internalState.browser = browser.new("Mozilla/3.0 (U; en)")
 --  internalState.browser = browser.new("FreePOPs/2.5 (U; en)")
@@ -601,15 +625,10 @@ function loginYahoo()
   local post
   local challengeCode, uVal
 
-  -- Handle rocketmail
+  -- Add the domain always
   --
-  if (domain == "rocketmail.com") then
-    domain = "yahoo.com"
-    username = username .. ".rm"
-  elseif (domain == "btinternet.com" or domain == "talk21.com") then
-    username = username .. "@" .. domain
-  end
-	
+  username = username .. "@" .. domain
+  
   -- DEBUG - Set the browser in verbose mode
   --
   -- browser:verbose_mode()
@@ -661,10 +680,31 @@ function loginYahoo()
     log.dbg( "login redirect response: err=" .. tostr(err), dbg_limit(body) )
   end
 
+  -- Check for interstitial page (advertisements)
+  -- 
+  url = browser:whathaveweread()
+  log.dbg("browser:whathaveweread()=" .. url)
+  str = string.match( url, '(http://.-interstitial.-/)[^/]*' )
+  if (str ~= nil) then
+    local str2 = string.match(body, '<a id="skip" href="([^"]+")>')
+	if (str2 ~= nil) then
+	  str2 = string.gsub(str2, "./", "")
+	  log.dbg("Found the link to get by interstitutial ad.")
+	  url = str .. str2
+	else
+	  log.dbg("Did not find the link to get by interstitutial ad.  Try one that worked at some point.")
+	  log.dbg("Page Body: " .. body)
+      url = str .. "mail_act.php?mpref=2" -- from skip() in http://us.i1.yimg.com/us.yimg.com/lib/tb7/mail_interstitial/mail_20080612.js	
+	end
+    log.dbg( "interstitial get: url=" .. url )
+    body, err = browser:get_uri(url)
+    log.dbg( "interstitial response: err=" .. tostr(err), dbg_limit(body) )
+    url = browser:whathaveweread()
+    log.dbg("browser:whathaveweread()=" .. url)
+  end
+
   -- Check for NewGUI and Try Beta
   local bNewGui = false
-  local url = browser:whathaveweread()
-  log.dbg("browser:whathaveweread()=" .. url)
   str = string.match(url, '/dc/try_mail')
   if (str == nil) then
     -- it's not a "try" page.  Check if its the new gui.
@@ -964,9 +1004,8 @@ function downloadYahooMsg(pstate, msg, n
   -- Get the header
   --
   if internalState.bNewGUI then
-    local msgid = internalState.msgids[uidl]
-    uidl = msgid
-    headers = getMsgHdr(pstate, uidl)
+    msgid = internalState.msgids[uidl]
+    headers = getMsgHdr(pstate, msgid)
   elseif (internalState.classicType == 'mc') then
     msgid = internalState.msgids[uidl]
 	hdrUrl = string.format(globals.strCmdAttach, internalState.strFileServer,
@@ -1038,6 +1077,22 @@ function downloadYahooMsg(pstate, msg, n
   if string.sub(headers,1,5) == "From " then
     headers = string.gsub(headers, "From .-\n", "", 1);
   end
+  
+  -- Add custom headers
+  --
+  local readStatus = "read"
+  if (internalState.unreadMsgs[uidl] == 1) then
+    readStatus = "unread"
+  end
+  local crlf
+  if (internalState.bNewGUI) then
+    crlf = "\n"
+    headers = string.gsub(headers, "\n+$", "")
+  else
+    crlf = "\r\n"
+    headers = string.gsub(headers, "\r\n\r\n$", "")
+  end
+  headers = headers .. crlf .. "X-FREEPOPS-READ-STATUS: " .. readStatus .. crlf .. crlf
 
   -- Remove "Content-Transfer-Encoding" line from the header.
   --
@@ -1063,7 +1118,7 @@ function downloadYahooMsg(pstate, msg, n
     if nLines == 0 then
       cbInfo.strText = ""
     else
-      getMsgBody(pstate, uidl, size, cbInfo)
+      getMsgBody(pstate, msgid, size, cbInfo)
     end
     mimer.pipe_msg(
       headers, 
@@ -1215,7 +1270,9 @@ function downloadYahooMsg(pstate, msg, n
 
   -- Do we need to mark the message as unread?
   --
-  if internalState.bNewGUI and internalState.bMarkMsgAsUnread == true then
+  if (internalState.bKeepMsgStatus == true) then
+    -- no op
+  elseif internalState.bNewGUI and internalState.bMarkMsgAsUnread == true then
     log.dbg("Marking as message: " .. uidl .. " as unread");
     markMsgUnread(uidl)
   elseif internalState.bNewGUI == false and internalState.classicType ~= 'mc' then
@@ -1420,7 +1477,7 @@ function getSTATList(pstate)
   local browser = internalState.browser
   local url = string.format(globals.strSoapCmd, internalState.strMailServer, 
     internalState.strWebSrvUrl, "ListMessages", internalState.strCrumb)
-  local nMaxMsgs = 999 
+  local nMaxMsgs = 9999 
   if internalState.statLimit ~= nil then
     nMaxMsgs = internalState.statLimit
   end
@@ -1451,7 +1508,7 @@ function getSTATList(pstate)
 
   -- Parse the message id's and sizes
   --
-  if (ipairs == nil) then
+  if (ent == nil) then
     internalState.bStatDone = true
     return POPSERVER_ERR_OK
   end
@@ -1472,6 +1529,16 @@ function getSTATList(pstate)
           break
         end        
       end
+	  
+	  local flagElem = elem[1]
+	  if (flagElem["tag"] == "flags") then
+	    attrs = flagElem["attr"]
+		local readAttr = attrs["isRead"]
+		if (readAttr == "0") then
+		  log.dbg("Message: " .. uidl .. " is unread.")
+		  internalState.unreadMsgs[uidl] = 1
+		end
+	  end
 
       -- Save the information
       --
@@ -1525,6 +1592,8 @@ function getMsgHdr(pstate, uidl)
 end
 
 function getMsgBody(pstate, uidl, size, cbInfo)
+  log.dbg("YahooNew getMsgBody - Entering")
+
   local browser = internalState.browser
   local url = string.format(globals.strSoapCmd, internalState.strMailServer, 
     internalState.strWebSrvUrl, "GetMessage", internalState.strCrumb)
@@ -1540,50 +1609,70 @@ function getMsgBody(pstate, uidl, size, 
 
   -- Get the parts
   --
-  local part = nil
-  local major = nil
+  
+--  log.dbg("YahooNew getMsgBody ent[3]=",tostr(ent[3]))
+
   for i, elem in ipairs (ent[3]) do
     if (type(elem) == "table" and elem["tag"] == "part") then
-      part = elem
       local attrs = elem["attr"]
       local partId = attrs["partId"]
       local type = attrs["type"]
       local subtype = attrs["subtype"]
+      local textElem = elem[1]
+      
+      local idPlain = nil
+      local idHtml = nil
+
 --      log.dbg("YahooNew getMsgBody partId="..tostr(partId)..
 --        ", type="..tostr(type)..", subtype="..tostr(subtype)..
+--        ", textElem="..tostr(textElem)..
 --        ", attrs=",tostr(attrs))
-      -- partId=1 is usually the main text (1.1) and html (1.2),
-      --  which is handled above and therefore not included in attachments.
-      -- don't assume any other plain or html is the main text
-      if (subtype == "plain") then
-        local textElem = elem[1]
+
+      -- The main body plain-text and html are handled by the first
+      --  two tests and should not be included in attachments.
+      -- multipart/alternative results in two parts: text (1), html (2)
+      -- if there are more parts (e.g. attachments), the multi/alt body
+      --  is usually within the first part, so: text (1.1) and html (1.2)
+      -- Don't assume any other plain or html is part of the main body
+
+      if (cbInfo.strText == nil) and
+          textElem and textElem["tag"] == "text" and
+          (subtype == "plain" or
+            (type == "text" and subtype == "")) then
+        idPlain = partId
+        log.dbg("YahooNew getMsgBody: adding plain text body = "..tostr(idPlain))
         local text = textElem[1]
         if (text == nil) then
           text = ""
         end
         text = text .. "\n"
         cbInfo.strText = getMsgCallBack(cbInfo, text)
-        -- remove major part from attachments
-        major = string.match(partId,"(.-)%.")
-        major = major or partId
-        cbInfo.attachments[major] = nil
-      elseif (subtype == "html") then
-        local textElem = elem[1]
+        -- remove part from attachments
+        cbInfo.attachments[idPlain] = nil
+
+      elseif (cbInfo.strHtml == nil) and
+        textElem and textElem["tag"] == "text" and
+          subtype == "html" then
+        idHtml = partId
+        log.dbg("YahooNew getMsgBody: adding html body = "..tostr(idHtml))
         local text = textElem[1]
         text = string.gsub(text, "&amp;", "&")
 --        text = string.gsub(text, "(</[^>]+>) ", "%1\r\n") 
         text = text .. "\n"
         cbInfo.strHtml = getMsgCallBack(cbInfo, text)
-        -- remove major part from attachments
-        major = string.match(partId,"(.-)%.")
-        major = major or partId
-        cbInfo.attachments[major] = nil
+        -- remove part from attachments
+        cbInfo.attachments[idHtml] = nil
+
       elseif (partId == "HEADER" or partId == "TEXT") then
+        log.dbg("YahooNew getMsgBody: partId ignored: "..tostr(partId))
         -- no-op
+
       else
         -- Only add to attachments once, by partId
-        -- We want only whole parts, not subparts (so no ".")
-        if (string.match(partId,"%.") == nil) then
+        -- We only want the leaves, not the multipart groupings
+        if (type ~= "multipart") and
+          (partId ~= idPlain) and (partId ~= idHtml) then
+          log.dbg("YahooNew getMsgBody: adding an attachment = "..tostr(partId))
           local file = attrs["dispParams"]
           local contentId = attrs["contentId"]
           if (file ~= nil) then
@@ -1592,15 +1681,13 @@ function getMsgBody(pstate, uidl, size, 
             url = string.format(globals.strCmdAttach, internalState.strMailServer,
                internalState.strMBox, escUidl, partId, internalState.strMailServer)
             cbInfo.attachments[partId] = getRealAttachmentUrl(url)
-  --          cbInfo.attachments[file] = getRealAttachmentUrl(url)
-  --          table.insert(cbInfo.attachments, cbInfo.attachments[file])
             -- an empty contentId is not a valid contentId
             if ((contentId ~= nil) and (contentId ~= "")) then
               contentId = string.sub(contentId, 2, -2)
               cbInfo.inlineids[partId] = contentId
-  --            cbInfo.inlineids[file] = contentId
-  --            table.insert(cbInfo.inlineids, table.getn(cbInfo.inlineids) + 1, contentId)
             end
+          else
+            log.dbg("YahooNew getMsgBody: nil dispParams -- attachment NOT ADDED.")
           end
         end
       end
@@ -1723,7 +1810,16 @@ function user(pstate, username)
   local domain = freepops.get_domain(username)
   local user = freepops.get_name(username)
 
-  internalState.strDomain = domain
+  -- Override the domain variable if it is set in the login parameter
+  --
+  local val = (freepops.MODULE_ARGS or {}).domain or nil
+  if val ~= nil then
+    log.dbg("Yahoo: Using overridden domain: " .. val)
+    internalState.strDomain = val
+  else
+    internalState.strDomain = domain
+  end
+
   internalState.strUser = user
 
   -- Figure out the domain specific flags
@@ -1850,6 +1946,15 @@ function user(pstate, username)
     internalState.statLimit = tonumber(val)
   end
 
+  -- If the flag keepmsgstatus=1 is set, then we won't touch the status of 
+  -- messages that we pull.
+  --
+  local val = (freepops.MODULE_ARGS or {}).keepmsgstatus or 0
+  if val == "1" then
+    log.dbg("Yahoo: All messages pulled will have its status left alone.")
+    internalState.bKeepMsgStatus = true
+  end
+
   return POPSERVER_ERR_OK
 end
 
@@ -2093,8 +2198,7 @@ function stat(pstate)
   local nPage = 0
   local nMsgs = 0
   local cmdUrl = string.format(strCmd, internalState.strMailServer,
-    internalState.classicType, internalState.strMBox, nPage, internalState.strView, 
-	internalState.strView);
+    internalState.classicType, internalState.strMBox, nPage, internalState.strView);
 
   -- Keep a list of IDs that we've seen.  With yahoo, their message list can 
   -- show messages that we've already seen.  This, although a bit hacky, will
@@ -2121,16 +2225,18 @@ function stat(pstate)
     -- Find out if there are any messages
     -- 
     local nomesg = string.match(body, globals.strMsgListNoMsgPat)
-    if (nomesg == nil) then
+	local nomesgMC = string.match(body, globals.strMsgListNoMsgPatMC)
+    if (nomesg == nil and internalState.classicType ~= "mc") then
       return true, nil
+	elseif (nomesgMC ~= nil and internalState.classicType == "mc") then
+	  return true, nil
     end
 
     -- Find only the HTML containing the message list
     --
     local subBody = string.match(body, globals.strMsgListHTMLPattern)
     if (subBody == nil) then
-      log.say("Yahoo Module needs to fix it's message list pattern matching.")
-      return false, nil
+      return true, nil
     end
 
     -- Tokenize out the message ID and size for each item in the list
@@ -2163,7 +2269,8 @@ function stat(pstate)
       -- Get the message id.  It's a series of a numbers followed by
       -- an underscore repeated.  
       --
-      msgid = string.match(msgid, globals.strMsgIDPattern) or string.match(msgid, globals.strMsgIDMCPattern) 
+      msgid = string.match(msgid, globals.strMsgIDPattern) or 
+	    string.match(msgid, globals.strMsgIDMCPattern) or string.match(msgid, globals.strMsgIDMIPattern)
       local uidl = string.gsub(msgid, "_[^_]-_[^_]-_", "_000_000_", 1);
       uidl = string.sub(uidl, 1, 60)
 
@@ -2179,11 +2286,17 @@ function stat(pstate)
       -- First figure out the unit (KB or just B)
       --
       local kbUnit = string.match(size, "([Kk])")
-      size = string.match(size, "([%d]+)[KkbB]")
-      if not kbUnit then 
-        size = math.max(tonumber(size), 0)
-      else
+	  local mbUnit = string.match(size, "([Mm])")
+      size = string.match(size, "([%d]+)[ ]-[KkMmbB]")
+	  if size == nil then
+	    size = 1024
+	  end
+      if kbUnit then 
         size = math.max(tonumber(size), 0) * 1024
+  	  elseif mbUnit then
+        size = math.max(tonumber(size), 0) * 1024 * 1024
+      else
+        size = math.max(tonumber(size), 0)
       end
 
       -- Save the information
@@ -2198,6 +2311,17 @@ function stat(pstate)
         internalState.msgids[uidl] = msgid
        end
     end
+	
+	log.dbg("Looking for unread messages.")
+	for clazz, uidl in string.gfind(body, globals.strMCUnreadPattern) do
+	  uidl = string.gsub(uidl, "/", "%%2F")
+      uidl = string.gsub(uidl, "_[^_]-_[^_]-_", "_000_000_", 1);
+      uidl = string.sub(uidl, 1, 60)
+	  if (clazz == "msgnew") then
+		  log.dbg("Message: " .. uidl .. " is unread.")
+		  internalState.unreadMsgs[uidl] = 1
+	  end
+	end
 		
     return true, nil
   end 
@@ -2206,7 +2330,7 @@ function stat(pstate)
   -- change the command url
   --
   local function funcCheckForMorePages(body) 
-    if internalState.statLimit ~= nil and internalState.statLimit >= nMsgs then
+    if internalState.statLimit ~= nil and internalState.statLimit <= nMsgs then
       return true
     end
 
@@ -2218,6 +2342,12 @@ function stat(pstate)
       cmdUrl = internalState.strMailServer .. nextURL
       return false
     else
+	  nextURL = string.match(body, globals.strMsgListNextPagePatternMC)
+	  if (nextURL ~= nil) then
+	    cmdUrl = internalState.strMailServer  .. internalState.classicType .. "/" .. nextURL
+	    return false
+	  end
+	
       return true
     end
   end