--[[ @Title: FTP Website Manager @Author: Jane Taubman & Mike Tate @Version: 1.3 @LastUpdated: 05 Feb 2015 @Description: Uploads Family Historian Generated Websites to an FTP site When run it builds a database of the pages and on a second run only uploads the pages which have changed. Note: If you include the current date in the pages this will class as all pages having changed. V1.1 Fix Module Loader problem V1.2 If one File Fails give option to continue upload. V1.3 Change ftp.lua module to use "PASV" for rootsweb. Fix "DELE" source 'no such file' error. Fix 'Cancel' from settings prompt. Handle subfolders. Allow prime ' in settings. Other minor fixes. All commented with -- V1.3 ]] -- debug = true setting = {} -- Holds the settings for Source Directory and FTP Host, Folder, Username, Password, and Trace function main() local dbenv,dbcon = opendb(fhGetPluginDataFileName()) setting = loadSettings(dbcon) local bclose = false local iButton while not(bclose) do if lfs.attributes(setting.directory,"mode") ~= "directory" then -- V1.3 warn that directory is missing fhMessageBox(setting.directory.."\n\nDirectory does not exist, and if not corrected,\nthen all FTP server files will get deleted!") end if setting.new then local promptMessage = 'Please use the Change Settings button to set up.' iButton = iupButtons('FTP Website Manager',promptMessage,'V','Change Settings','Help','Exit') if iButton > 0 then iButton = iButton + 2 end else local directory = setting.directory if #directory > 60 then directory = '...'..directory:sub(-60) -- V1.3 if more than 60 chars prefix truncated directory with elipsis end local promptMessage = string.format('From:\n %s\nTo:\n %s/%s',directory,setting.host,setting.folder) iButton = iupButtons('FTP Website Manager',promptMessage,'V','Update Site','Clear Page History','Change Settings','Help','Exit') end if iButton == 1 then local list = buildpagelist(dbcon,setting) if list then if #list == 0 then fhMessageBox('No files found to update') return else local ret = uploadFiles(dbcon,list) local msg = ' Files Updated' if #list == 1 then msg = msg:gsub('Files','File') end fhMessageBox(#list..msg) end end elseif iButton == 2 then resetDatabase(dbcon) elseif iButton == 3 then settingsPrompt(dbcon) elseif iButton == 4 then -- Show Help Window GUI_HelpDialogue() elseif iButton == 5 or iButton == 0 then bclose = true end end closedb(dbenv,dbcon) end function settingsPrompt(dbcon) local ret local settings = {} for a,b in pairs (setting) do settings[a] = b end -- V1.3 preserve settings in case Cancel selected local defdir = setting.directory or "" if lfs.attributes(defdir,"mode") ~= "directory" then -- V1.3 restore default if setting.directory missing defdir = default end ret,setting.directory,setting.host,setting.folder,setting.userid,setting.password,setting.trace = iup.GetParam("FTP Website Manager - Settings", iup.NULL, "Source Folder: %f[DIR||"..defdir.."]\n".. -- V1.3 correct iupFileDlg params "FTP Host URL: %s\n".. "FTP Folder: %s\n".. "FTP Username: %s\n".. "FTP Password: %s\n".. "FTP Trace: %b[ No thanks , Yes please ]\n", setting.directory,setting.host,setting.folder,setting.userid,setting.password,setting.trace) if (ret == true) then setting.directory = setting.directory:gsub('/','\\'):gsub('\\+$','') -- V1.3 change all / to \ and erase all trailing \ setting.folder = setting.folder:gsub('\\','/'):gsub('/+$','') -- V1.3 change all \ to / and erase all trailing / dump(setting.directory) saveSettings(dbcon,setting) setting.new = nil else for a,b in pairs (settings) do setting[a] = b end -- V1.3 restore original settings after Cancel end end function uploadFiles(dbcon,list) -- Connect to FTP server ProgressDisplay.Start('Updating Files on FTP Server',100) local ipos = 0 local step = 1/#list * 100 local bQuit = false for i,filedata in pairs(list) do local tofile = filedata.filename:gsub(setting.directory..'\\','') -- V1.3 remove '\' between folder and filename ipos = ipos + 1 ProgressDisplay.SetMessage(filedata.action..': '..tofile..' ('..ipos..'/'..#list..')') ProgressDisplay.Step(step) if ProgressDisplay.Cancel() then break end local status = ftpPutFile(filedata.filename,filedata.action) -- V1.3 do not need tofile param if status == 'OK' then updRecord(dbcon,filedata.filename,filedata.md5hash) else local strMsg = 'An error occured updating: '..tofile..'\n'..status..'\nDo you want to continue with other files?' local a = fhMessageBox(strMsg,'MB_YESNO','MB_ICONQUESTION') if a == 'No' then bQuit = true end end if bQuit then break end end ProgressDisplay.Reset() ProgressDisplay.Close() end function ftpPutFile(fromfile,action) -- V1.3 do not need tofile param, use file instead local actions = {update='STOR',delete='DELE'} local path, file, ext = SplitFilename(fromfile) local stype = 'i' if ext == 'html' or ext == 'css' or ext == 'js' then -- Use Binary for everything other than the html and css files. stype = 'a' end if action == 'delete' then fromfile = 'nul' end -- V1.3 avoid 'no such file error' for source local p = { host = setting.host, user = setting.userid, password = setting.password, command = actions[action], argument = setting.folder..'/'..file, -- V1.3 use file instead of tofile type = stype, source = ltn12.source.file(io.open(fromfile, "rb")) } local f, e = ftp.put(p) showtrace('ftp.put',p,setting.trace) return e or 'OK' end -- Update record for new files function updRecord(dbcon,filename,md5hash) -- V1.3 use "%s" to allow prime ' in filename local sql = string.format('UPDATE pagelist set md5hash = "%s" where filename="%s"',md5hash,filename) local res = assert(dbcon:execute(sql)) end function getFTPList(setting) local t = {} local g = { host = setting.host, sink = ltn12.sink.table(t), user = setting.userid, password = setting.password, argument = setting.folder, -- V1.3 use argument instead of appending to command command = 'NLST', type = 'a' } local _, e = ftp.get(g) showtrace('ftp.get',g,setting.trace) if e then return nil,e else local t2 = table.concat(t) local t3 = split(t2,'\r\n') return t3 end end function showtrace(func,ftp,trace) -- V1.3 tp trace debug listing local strTrace = '\nOperation:\t'..func..' '..ftp.command..' '..ftp.argument..'\n' strTrace = strTrace..table.concat(tp.GetTrace(),'\n') strTrace = strTrace:gsub('Response:','Response: ') if trace == 1 then if fhMessageBox('FTP Website Manager - Trace Log\n'..strTrace,'MB_OKCANCEL') == 'Cancel' then setting.trace = 0 end end print(strTrace) end function buildpagelist(dbcon,setting) local function fliptable(table) local newtable = {} for i,v in pairs(table) do v = v:gsub(setting.folder.."/","") -- V1.3 remove folder from filename newtable[v] = i end return newtable end local function chkrecord(filename) -- Add records for new files -- V1.3 use "%s" to allow prime ' in filename local sql = string.format('INSERT OR IGNORE INTO pagelist (filename,md5hash) VALUES ("%s"," ")',filename) local res = assert(dbcon:execute(sql)) end local function getHash(filename) -- Add records for new files -- V1.3 use "%s" to allow prime ' in filename local sql = string.format('select md5hash from pagelist where filename = "%s"',filename) local cur = assert(dbcon:execute(sql)) local row = cur:fetch() cur:close() return row end local basedir = setting.directory local tblList = {} local tblAll = {} ProgressDisplay.Start('Building Transfer List',100) ProgressDisplay.SetMessage('Retrieving List From FTP Server') local ftplist, e = getFTPList(setting) if e then fhMessageBox('Error Occured Connecting to FTP site. Please check your settings\n'..e) ProgressDisplay.Reset() ProgressDisplay.Close() return end ftplist = fliptable(ftplist) -- Quick Count Files local icount = 0 for filename, attr in dirtree(basedir) do local path, file, ext = SplitFilename(filename) -- V1.3 only count contents of basedir and not sub-folders if path == basedir..'\\' and attr.mode == 'file' then icount = icount + 1 end -- V1.3 correct istep to icount, and only count files not folders end local istep = 1 / icount * 100 ProgressDisplay.SetMessage(#tblList..' files found to update... ') for filename, attr in dirtree(basedir) do local path, file, ext = SplitFilename(filename) -- V1.3 only check contents of basedir and not sub-folders if path == basedir..'\\' then if attr.mode == 'file' then -- V1.3 handle files here ProgressDisplay.Step(istep) if ProgressDisplay.Cancel() then break end ProgressDisplay.SetMessage(#tblList..' files found to update... Checking: '..file) chkrecord(filename) -- Add if needed local oldhash = getHash(filename) -- Remove file from FTP List as it does not need deleting ftplist[file] = nil -- compute MD5 routine local md5hash = md5.sumhexa(LoadFromFile(filename)) if oldhash ~= md5hash then table.insert(tblList,{filename=filename, md5hash=md5hash, action='update'}) ProgressDisplay.SetMessage(#tblList..' files found to update... ') end else ftplist[file] = nil -- V1.3 remove matching folders from FTP List to prevent deletion end end end for i,v in pairs(ftplist) do -- In the Delete List remove any files starting with . and any folders if i:sub(1,1) ~= '.' then local path, file, ext = SplitFilename(i) if file ~= ext then -- V1.3 exclude folders to prevent deletion chkrecord(file) -- Add if needed table.insert(tblList,{filename=file, md5hash='0', action='delete'}) -- V1.3 only need filename for Delete end end end ProgressDisplay.Reset() ProgressDisplay.Close() return tblList end function resetDatabase(dbcon) local sql = 'delete from pagelist' local res = assert(dbcon:execute(sql)) fhMessageBox('Page Database cleared') end function opendb(dbname) -- Check for Settings Database and create if needed local db = fhGetPluginDataFileName() local dbenv = assert (luasql.sqlite3()) -- connect to data source, if the file does not exist it will be created local dbcon = assert (dbenv:connect(db)) -- check table for page list checkTable(dbcon,'pagelist', [[CREATE TABLE pagelist(filename varchar(500), md5hash varchar(32), UNIQUE (filename)) ]]) -- create table for settings checkTable(dbcon,'settings', [[CREATE TABLE settings(key varchar(20), directory varchar(500), host varchar(500), folder varchar(50), userid varchar(50), password varchar(50), UNIQUE (key)) ]]) return dbenv,dbcon end function checkTable(dbcon,table,createString) local sql = string.format([[SELECT count(name) as count FROM sqlite_master WHERE type='table' AND name='%s']],table) local cur = assert(dbcon:execute(sql)) local rowcount = cur:fetch (row, "a") cur:close() if tonumber(rowcount) == 0 then -- Table not found create it local res,err = assert(dbcon:execute(createString)) end end function closedb(dbenv,dbcon) dbcon:close() dbenv:close() end function loadSettings(dbcon) local sql = [[SELECT * FROM settings]] local cur,err = assert(dbcon:execute(sql)) local row = cur:fetch({},'a') cur:close() if row then for item, text in pairs (row) do if type(text) == "string" then row[item] = text:gsub("\127","'") -- V1.3 convert delete char back to prime ' char end end row.trace = 0 return row else -- return default values return { directory = default, -- V1.3 default to Project...\Public\FH Website host = 'websitehost', folder = '/', userid = 'user', password = 'password', trace = 0, new = 'yes' } end end function saveSettings(dbcon,settings) local row = {} for item, text in pairs (settings) do if type(text) == "string" then row[item] = text:gsub("'","\127") -- V1.3 prime ' not allowed in sql values so convert to delete char end end -- Check for Settings if row.new == 'yes' then -- Create sql = string.format([[insert into settings (directory, host, folder, userid, password, trace) Values('%s','%s','%s','%s','%s')]],row.directory,row.host,row.folder,row.userid,row.password) else -- Update sql = string.format([[update settings set directory = '%s', host = '%s', folder = '%s', userid = '%s', password = '%s']],row.directory,row.host,row.folder,row.userid,row.password) end local res = assert(dbcon:execute(sql)) end function split(str, pat) local t = {} -- NOTE: use {n = 0} in Lua-5.0 local fpat = "(.-)" .. pat local last_end = 1 local s, e, cap = str:find(fpat, 1) while s do if s ~= 1 or cap ~= "" then table.insert(t,cap) end last_end = e+1 s, e, cap = str:find(fpat, last_end) end if last_end <= #str then cap = str:sub(last_end) table.insert(t, cap) end return t end function dirtree(dir) assert(dir and dir ~= "", "directory parameter is missing or empty") if string.sub(dir, -1) == "/" then dir=string.sub(dir, 1, -2) end local function yieldtree(dir) for entry in lfs.dir(dir) do if entry ~= "." and entry ~= ".." then entry=dir.."\\"..entry local attr=lfs.attributes(entry) coroutine.yield(entry,attr) if attr.mode == "directory" then yieldtree(entry) end end end end return coroutine.wrap(function() yieldtree(dir) end) end --[[ @Title: Progress Display (drop in) @Author: Jane Taubman / Mike Tate @LastUpdated: May 2012 @Description: Allows easy adding of a Progress Bar to any long running Plugin ]] ProgressDisplay = { Start = function(strTitle,intMax) -- Create and start the Progress Display window controls local StrWhite = "255 255 255" if not dlgProgress then cancelflag = false local cancelbutton = iup.button { title="Cancel", rastersize="200x30", action = function() cancelflag = true -- Signal that Cancel button was pressed return iup.CLOSE end } gaugeProgress = iup.progressbar { rastersize="400x30", max=intMax } -- Set progress bar maximum range messageline = iup.label { title=" ", expand="YES", alignment="ACENTER" } dlgProgress = iup.dialog { title=strTitle, dialogframe="YES", background=strWhite, -- Remove Windows minimize/maximize menu iup.vbox { alignment="ACENTER", gap="10", margin="10x10", messageline, gaugeProgress, cancelbutton } } dlgProgress.close_cb = cancelbutton.action -- Windows Close button acts as Cancel button dlgProgress:showxy(iup.CENTER, iup.CENTER) -- Show the Progress Display dialogue window end end, SetMessage = function(strMessage) -- Set the progress message if dlgProgress then messageline.title = strMessage end end, Step = function(iStep) -- Step the Progress Bar forward if dlgProgress then gaugeProgress.value = gaugeProgress.value + iStep local val = tonumber(gaugeProgress.value) local max = tonumber(gaugeProgress.max) if val > max then gaugeProgress.value = 0 end iup.LoopStep() end end, Reset = function() -- Reset progress bar if dlgProgress then gaugeProgress.value = 0 end end, Cancel = function() -- Check if Cancel button pressed return cancelflag end, Close = function() -- Close the dialogue window cancelflag = false if dlgProgress then dlgProgress:destroy() dlgProgress = nil end end, } -------------------------------------------------------------------------------------------- End Progress Bar -- Open File and return Handle -- function OpenFile(strFileName,strMode) local fileHandle, strError = io.open(strFileName,strMode) if not fileHandle then error("\n Unable to open file in \""..strMode.."\" mode. \n "..strFileName.." \n "..tostring(strError).." \n") end return fileHandle end -- function OpenFile -- Load string from file -- function LoadFromFile(strFileName) local fileHandle = OpenFile(strFileName,"rb") local strString = fileHandle:read("*all") assert(fileHandle:close()) return strString end -- function LoadFromFile -- Save string to file -- function SaveStringToFile(strString,strFileName) local fileHandle = OpenFile(strFileName,"w") fileHandle:write(strString) assert(fileHandle:close()) end -- function SaveStringToFile -- Return the Path, Filename, and extension as 3 values function SplitFilename(strFilename) return strFilename:match("(.-)([^\\]-([^\\%.]+))$") end -- function SplitFilename -- Load Module function loadrequire(module,extended) if not(extended) then extended = module end local function installmodule(module,filename) local bmodule = false if not(filename) then filename = module..'.mod' bmodule = true end local storein = fhGetContextInfo('CI_APP_DATA_FOLDER')..'\\Plugins\\' -- Check if subdirectory needed local path = string.match(filename, "(.-)[^/]-[^%.]+$") if path ~= "" then path = path:gsub('/','\\') -- Create sub-directory lfs.mkdir(storein..path) end -- Get file down and install it local http = luacom.CreateObject("winhttp.winhttprequest.5.1") local url = "http://www.family-historian.co.uk/lnk/getpluginmodule.php?file="..filename http:Open("GET",url,false) http:Send() http:WaitForResponse(30) local status = http.StatusText if status == 'OK' then length = http:GetResponseHeader('Content-Length') data = http.ResponseBody if bmodule then local modlist = loadstring(http.ResponseBody) for _,f in pairs(modlist()) do if not(installmodule(module,f)) then break end end else local function OpenFile(strFileName,strMode) local fileHandle, strError = io.open(strFileName,strMode) if not fileHandle then error("\n Unable to open file in \""..strMode.."\" mode. \n "..strFileName.." \n "..tostring(strError).." \n") end return fileHandle end -- OpenFile local function SaveStringToFile(strString,strFileName) local fileHandle = OpenFile(strFileName,"wb") fileHandle:write(strString) assert(fileHandle:close()) end -- SaveStringToFile SaveStringToFile(data,storein..filename) end return true else fhMessageBox('An error occurred in Download please try later') return false end end local function requiref(module) require(module) end local res = pcall(requiref,extended) if not(res) then local ans = fhMessageBox( 'This plugin requires '..module..' support, please click OK to download and install the module', 'MB_OKCANCEL','MB_ICONEXCLAMATION') if ans ~= 'OK' then return false end if installmodule(module) then package.loaded[extended] = nil -- Reset Failed Load require(extended) else return false end end return true end --[[ @Title: User Interface Buttons Snippet @Author: Mike Tate / Jane Taubman @LastUpdated: May 2012 @Version: 1.4 @Description: GUI dialogue for multiple buttons @params strTitle: Title of Message Box strMessage: Message to show above buttons strBoxType: Either "H" for Horizontal buttons or "V" for Vertical ones. ... : All other parameters will be treated as button titles. ]] function iupButtons(strTitle,strMessage,strBoxType,...) local intButton = 0 -- Returned value if X Close button is used -- Create the GUI labels and buttons local lblMessage = iup.label{title=strMessage,expand="YES"} local lblLineSep = iup.label{separator="HORIZONTAL"} local iupBox = iup.hbox{homogeneous="YES"} if strBoxType == "V" then iupBox = iup.vbox{homogeneous="YES"} end for intArgNum, strButton in ipairs(arg) do local btnName = iup.button{title=strButton,expand="YES",padding="4",action=function() intButton=intArgNum return iup.CLOSE end} iup.Append(iupBox,btnName) end -- Create dialogue and turn off resize, maximize, minimize, and menubox except Close button local dialogue = iup.dialog{title=strTitle,iup.vbox{lblMessage,lblLineSep,iupBox},dialogframe="YES",background="250 250 250",gap="8",margin="8x8"} dialogue:show() if (iup.MainLoopLevel()==0) then iup.MainLoop() end dialogue:destroy() return intButton end -- function iupButtons --[[ @Title: Display FHUG Wiki Help Snippet @Author: Mike Tate @LastUpdated: May 2012 @Version: 1.1 @Description: Displays FHUG Wiki help pages in a popup GUI. ]] -- GUI Help & Advice Dialogue -- function GUI_HelpDialogue() local StrPlugin = "Display FHUG Wiki" -- Plugin title & version local StrIssue = " V1.0" local StrRed = "255 000 000" -- Color attributes local StrGreen = "000 120 000" local StrBlue = "000 000 255" local StrBlack = "000 000 000" local StrWhite = "255 255 255" local StrBigMargin = "10x10" -- Layout attributes local StrMinMargin = "1x1" local StrGap = "10" local StrFontFace = string.gsub(iup.GetGlobal("DEFAULTFONT"),",.*","") local StrFontHead = StrFontFace..", Bold -16" local StrFontBody = StrFontFace..", -16" local StrFHUG = "http://www.fhug.org.uk/wiki/wiki/doku.php?id=" local function doActivateMainHelpButton() if BtnMainHelp then BtnMainHelp.active = "YES" end end -- Create the WebBrowser based on its ProgID and connect it to LuaCOM local oleControl = iup.olecontrol{ "Shell.Explorer.1", designmode="NO", } oleControl:CreateLuaCOM() -- Create each GUI button with title and tooltip local btnClose = iup.button { title="Close this Window" , tip="Close this Help and Advice window" } -- The following control is global to allow Main GUI to alter font HboxHelp = iup.hbox { font=StrFontBody, margin=StrMinMargin, homogeneous="YES", btnMain, btnPath, btnClose, } local strExpChild = "NO" local iupVersion = iup.GetGlobal("VERSION") if iupVersion == "3.5" then strExpChild = "YES" end -- for IUP 3.11.2 local dialogHelp = iup.dialog { title=StrPlugin.." Help & Advice", background=StrWhite, startfocus=btnClose, rastersize="1000x700", iup.vbox { alignment="ACENTER", gap=StrGap, margin=StrBigMargin, expandchildren=strExpChild, oleControl, HboxHelp, }, close_cb=function() doActivateMainHelpButton() end, } local strFHUG = StrFHUG.."plugins:help:" -- Set other GUI control attributes for iupName, tblAttr in pairs( { -- Control= 1~fgcolor , 2~Navigate URL , 3~action function() [btnClose]= { StrRed , false , function() dialogHelp:destroy() doActivateMainHelpButton() return iup.CLOSE end }, } ) do iupName.expand = "HORIZONTAL" iupName.size = "x10" iupName.fgcolor = tblAttr[1] if tblAttr[2] then iupName.action = function() oleControl.com:Navigate(tblAttr[2]) end end if tblAttr[3] then iupName.action = tblAttr[3] end end dialogHelp:show() dialogHelp.rastersize=iup.NULL -- Allow window to be resized oleControl.com:Navigate(strFHUG.."ftp_website_manager") if (iup.MainLoopLevel()==0) then iup.MainLoop() end end -- function GUI_HelpDialogue function dump (tt, indent, done, label) if label == nil then label = 'Dump' end if debug == true then done = done or {} indent = indent or 0 if type(tt) == "table" then if indent == 0 then io.write(string.rep (" ", indent)) io.write(label..'\n') end for key, value in pairs (tt) do io.write(string.rep (" ", indent)) -- indent it if type (value) == "table" and not done [value] then done [value] = true io.write(string.format("[%s] => table\n", tostring (key))); io.write(string.rep (" ", indent+4)) -- indent it io.write("(\n"); dump (tostring(key),value, indent + 7, done) io.write(string.rep (" ", indent+4)) -- indent it io.write(")\n"); else io.write(string.format("[%s] => %s\n", tostring (key), tostring(value))) end end else io.write(label..':'..tostring(tt)) end else return end end ------------------------------------------------------ End of Functions require 'lfs' require 'luacom' require 'iupluaole' if not(loadrequire('luasql','luasql.sqlite3')) then return end if not(loadrequire('md5')) then return end if not(loadrequire('socket')) then return end local socket = fhGetContextInfo('CI_APP_DATA_FOLDER')..'\\Plugins\\socket' local module = socket..'\\tp.lua' local data = LoadFromFile(module) if not data:match('Trace') then -- V1.3 add tp trace debug feature data = data:gsub('(TIMEOUT = 60)', '%1\n Trace = {}\n function GetTrace() local t={} for i,v in base.ipairs(Trace) do t[i]=v end Trace={} return t end' ) data = data:gsub('(local reply = line)', '%1\n if Trace then Trace[#Trace+1] = "Response:\t"..(err or reply) end' ) data = data:gsub('(function metat%.__index:command%(cmd, arg%))', '%1\n if Trace then Trace[#Trace+1] = "Command:\t"..cmd.." "..(arg or "") end' ) data = data:gsub('(function connect%(host, port, timeout, create%))', '%1\n if Trace then Trace[#Trace+1] = "Connecting:\t"..host.."\tPort: "..port end' ) SaveStringToFile(data,module) module = socket..'\\ftp.lua' data = LoadFromFile(module) data = data:gsub('"pasv"','"PASV"') -- V1.3 fix ftp module PASV for rootsweb SaveStringToFile(data,module) elseif data:match('return Trace') then data = data:gsub('return Trace','local t={} for i,v in base.ipairs(Trace) do t[i]=v end Trace={} return t') SaveStringToFile(data,module) end ftp = require("socket.ftp") tp = require("socket.tp") default = fhGetContextInfo('CI_PROJECT_PUBLIC_FOLDER') -- V1.3 default Source Folder if lfs.attributes(default..'\\FH Website',"mode") == "directory" then default = default..'\\FH Website' end main()