Clean Unwanted Fields.fh_lua--[[
@Title: Clean Unwanted Fields
@Type: Standard
@Author: Mike Tate
@Contributors:
@Version: 1.4
@Keywords:
@LastUpdated: 16 Feb 2026
@Licence: This plugin is copyright (c) 2026 Mike Tate & contributors and is licensed under the MIT License which is hereby incorporated by reference (see https://pluginstore.family-historian.co.uk/fh-plugin-licence)
@Description: Removes fields that typically upset Ancestry, or hold sensitive data, or any chosen by user.
It is meant to be used in conjunction with the Export GEDCOM File and Split Tree Helper commands,
so will prompt for confirmation if you try to use it on a Gedcom which is open in Project mode.
@V1.4: Centre windows on FH window; Check for Updates button; Ch-ViS 1.4; progbar 3.1; FSO filename accents;
@V1.3: FH V7 Lua 3.5 IUP 3.28 compatible;
@V1.2: Cater for data refs with instances such as %INDI.NAME[2]%, add a Result Set, ensure all versions use one Plugin Data .dat file.
@V1.1: Now remembers its screen position, handles specific Fact tags, and more rigorous data ref validation check.
@V1.0: First publication in the Plugin Store.
@V0.5: Add user dialogue and sticky Data Reference list.
@V0.4: ?
@V0.3: Third prototype with remove whole record example.
@V0.2: Second prototype redesigned and renamed to allow user edits.
@V0.1: First prototype was Clean Ancestry Fields.
]]
require("iuplua") -- To access IUP GUI library
require("luacom") -- To create File System Object -- V1.4
local FSO = luacom.CreateObject("Scripting.FileSystemObject")
local strVersion = "1.4"
local strPluginName = "Clean Unwanted Fields "..strVersion -- Update title and version number here
if fhGetAppVersion() > 5 then -- Cater for Unicode UTF-8 from FH Version 6 onwards
fhSetStringEncoding("UTF-8")
iup.SetGlobal("UTF8MODE","YES")
iup.SetGlobal("CUSTOMQUITMESSAGE","YES") -- Needed for IUP 3.28 -- V1.4
end
function Encoding()
if fhGetAppVersion() > 5 then return fhGetStringEncoding() end
return "ANSI"
end -- function Encoding
-- Split a string using "," or chosen separator --
function split(strTxt,strSep)
local tblFields = {}
local strPattern = string.format("([^%s]+)", strSep or ",")
strTxt = tostring(strTxt or "")
strTxt:gsub(strPattern, function(strField) tblFields[#tblFields+1] = strField end)
return tblFields
end -- function split
-- Report error message --
local function doError(strMessage,errFunction)
-- strMessage ~ error message text
-- errFunction ~ optional error reporting function
if type(errFunction) == "function" then
errFunction(strMessage)
else
error(strMessage)
end
end -- function doError
-- Convert filename to ANSI alternative and indicate success --
function FileNameToANSI(strFileName,strAnsiName)
-- strFileName ~ full file path
-- strAnsiFile ~ ANSI file name & type
-- return values ~ ANSI file path, true if original path was ANSI compatible
if Encoding() == "ANSI" then return strFileName, true end
local isFlag = fhIsConversionLossFlagSet()
fhSetConversionLossFlag(false)
local strAnsi = fhConvertUTF8toANSI(strFileName)
local wasAnsi = true
if fhIsConversionLossFlagSet() then
strAnsiName = strAnsiName or "ANSI.ANSI"
strAnsi = fhGetContextInfo("CI_APP_DATA_FOLDER").."\\Plugin Data\\"..strAnsiName
wasAnsi = false
end
fhSetConversionLossFlag(isFlag)
return strAnsi, wasAnsi
end -- function FileNameToANSI
-- Get parent folder --
function GetParentFolder(strFileName)
-- strFileName ~ full file path
-- return value ~ parent folder path
local strParent = FSO:GetParentFolderName(strFileName) --! Faulty in FH v6 with Unicode chars in path
if fhGetAppVersion() == 6 then
local _, wasAnsi = FileNameToANSI(strFileName)
if not wasAnsi then
strParent = strFileName:match("^(.+)[\\/][^\\/]+[\\/]?$")
end
end
return strParent
end -- function GetParentFolder
-- Check if file exists --
function FlgFileExists(strFileName)
-- strFileName ~ full file path
-- return value ~ true if it exists
return FSO:FileExists(strFileName)
end -- function FlgFileExists
-- Delete a file if it exists --
function DeleteFile(strFileName,errFunction)
-- strFileName ~ full file path
-- errFunction ~ optional error reporting function
-- return value ~ true if file does not exist or is deleted else false
if FSO:FileExists(strFileName) then
FSO:DeleteFile(strFileName,true)
if FSO:FileExists(strFileName) then
doError("File Not Deleted:\n"..strFileName.."\n",errFunction)
return false
end
end
return true
end -- function DeleteFile
-- Copy a file if it exists and destination is not a folder --
function CopyFile(strFileName,strDestination)
-- strFileName ~ full source file path
-- strDestination~ full target file path
-- return value ~ true if file exists and is copied else false
if MakeFolder(GetParentFolder(strDestination)) and FSO:FileExists(strFileName) and not FSO:FolderExists(strDestination) then
FSO:CopyFile(strFileName,strDestination)
if FSO:FileExists(strDestination) then
return true
end
end
return false
end -- function CopyFile
-- Move a file if it exists and destination is not a folder --
function MoveFile(strFileName,strDestination)
-- strFileName ~ full source file path
-- strDestination~ full target file path
-- return value ~ true if file exists and is moved else false
if MakeFolder(GetParentFolder(strDestination)) and FSO:FileExists(strFileName) and not FSO:FolderExists(strDestination) then
if DeleteFile(strDestination) then
FSO:MoveFile(strFileName,strDestination)
if FSO:FileExists(strDestination) then
return true
end
end
end
return false
end -- function MoveFile
-- Open File with ANSI path and return Handle --
function OpenFile(strFileName,strMode)
-- strFileName ~ full file path
-- strMode ~ "r", "w", "a" optionally suffixed with "+" &/or "b"
-- return value ~ file handle
local fileHandle, strError = io.open(strFileName,strMode)
if fileHandle == nil then
error("\n Unable to open file in \""..strMode.."\" mode. \n "..strFileName.." \n "..strError.." \n")
end
return fileHandle
end -- function OpenFile
-- Save string to file --
function SaveStringToFile(strContents,strFileName,strFormat)
-- strContents ~ text string
-- strFileName ~ full file path
-- strFormat ~ optional "UTF-8" or "UTF-16LE"
-- return value ~ true if successful else false
strFormat = strFormat or "UTF-8"
if fhGetAppVersion() > 6 then
return fhSaveTextFile(strFileName,strContents,strFormat)
end
local strAnsi, wasAnsi = FileNameToANSI(strFileName)
local fileHandle = OpenFile(strAnsi,"w")
fileHandle:write(strContents)
assert(fileHandle:close())
if not wasAnsi then
MoveFile(strAnsi,strFileName)
end
return true
end -- function SaveStringToFile
-- Load string from file --
function StrLoadFromFile(strFileName,strFormat)
-- strFileName ~ full file path
-- strFormat ~ optional "UTF-8" or "UTF-16LE"
-- return value ~ file contents
strFormat = strFormat or "UTF-8"
if fhGetAppVersion() > 6 then
return fhLoadTextFile(strFileName,strFormat)
end
local strAnsi, wasAnsi = FileNameToANSI(strFileName)
if not wasAnsi then
CopyFile(strFileName,strAnsi)
end
local fileHandle = OpenFile(strAnsi,"r")
local strContents = fileHandle:read("*all")
assert(fileHandle:close())
return strContents
end -- function StrLoadFromFile
--[[
@Module: +fh+progbar_v3
@Author: Mike Tate
@Version: 3.1
@LastUpdated: 23 Jan 2026
@Description: Progress Bar library module.
@V3.1: Use NATIVEPARENT amd CENTERPARENT.
@V3.0: Function Prototype Closure version.
@V1.0: Initial version.
]]
local function progbar_v3()
local fh = {} -- Local environment table
require "iuplua" -- To access GUI window builder
iup.SetGlobal("CUSTOMQUITMESSAGE","YES") -- Needed for IUP 3.28
local tblBars = {} -- Table for optional external attributes
local strBack = "255 255 255" -- Background colour default is white
local strBody = "0 0 0" -- Body text colour default is black
local strFont = nil -- Font dialogue default is current font
local strStop = "255 0 0" -- Stop button colour default is red
local intPosX = iup.CENTERPARENT -- Show window default position is central -- V3.1
local intPosY = iup.CENTERPARENT
local intMax, intVal, intPercent, intStart, intDelta, intScale, strClock, isBarStop
local lblText, barGauge, lblDelta, btnStop, dlgGauge
local function doFocus() -- Bring the Progress Bar window into Focus
dlgGauge.BringFront="YES" -- If used too often, inhibits other windows scroll bars, etc
end -- local function doFocus
local function doUpdate() -- Update the Progress Gauge and the Delta % with clock
barGauge.Value = intVal
lblDelta.Title = string.format("%4d %% %s ",math.floor(intPercent),strClock)
end -- local function doUpdate
local function doReset() -- Reset all dialogue variables and Update display
intVal = 0 -- Current value of Progress Bar
intPercent= 0.01 -- Percentage of progress
intStart = os.time() -- Start time of progress
intDelta = 0 -- Delta time of progress
intScale = math.ceil( intMax / 1000 ) -- Scale of percentage per second of progress (initial guess is corrected in Step function)
strClock = "00 : 00 : 00" -- Clock delta time display
isBarStop = false -- Stop button pressed signal
doUpdate()
doFocus()
end -- local function doReset
function fh.Start(strTitle,intMaximum) -- Create & start Progress Bar window
if not dlgGauge then
strTitle = strTitle or "" -- Dialogue and button title
intMax = intMaximum or 100 -- Maximun range of Progress Bar, default is 100
local strSize = tostring( math.max( 100, string.len(" Stop "..strTitle) * 8 ) ).."x30" -- Adjust Stop button size to Title
lblText = iup.label { Title=" "; Expand="YES"; Alignment="ACENTER"; Tip="Progress Message"; }
barGauge = iup.progressbar { RasterSize="400x30"; Value=0; Max=intMax; Tip="Progress Bar"; }
lblDelta = iup.label { Title=" "; Expand="YES"; Alignment="ACENTER"; Tip="Percentage and Elapsed Time"; }
btnStop = iup.button { Title=" Stop "..strTitle; RasterSize=strSize; FgColor=strStop; Tip="Stop Progress Button"; action=function() isBarStop = true end; } -- Signal Stop button pressed return iup.CLOSE -- Often caused main GUI to close !!!
dlgGauge = iup.dialog { Title=strTitle.." Progress "; Font=strFont; FgColor=strBody; Background=strBack; DialogFrame="YES"; -- Remove Windows minimize/maximize menu
iup.vbox{ Alignment="ACENTER"; Gap="10"; Margin="10x10";
lblText;
barGauge;
lblDelta;
btnStop;
};
move_cb = function(self,x,y) tblBars.X = x tblBars.Y = y end;
close_cb = btnStop.action; -- Windows Close button = Stop button
}
if type(tblBars.GUI) == "table"
and type(tblBars.GUI.ShowDialogue) == "function" then
dlgGauge.move_cb = nil -- Use GUI library to show & move window
tblBars.GUI.ShowDialogue("Bars",dlgGauge,btnStop,"showxy")
else
if fhGetAppVersion() > 6 then -- Window centres on FH parent -- V3.1
iup.SetAttribute(dlgGauge,"NATIVEPARENT",fhGetContextInfo("CI_PARENT_HWND"))
end
dlgGauge:showxy(intPosX,intPosY) -- Show the Progress Bar window
end
doReset() -- Reset the Progress Bar display
end
end -- function Start
function fh.Message(strText) -- Show the Progress Bar message
if dlgGauge then lblText.Title = strText end
end -- function Message
function fh.Step(intStep) -- Step the Progress Bar forward
if dlgGauge then
intVal = intVal + ( intStep or 1 ) -- Default step is 1
local intNew = math.ceil( intVal / intMax * 100 * intScale ) / intScale
if intPercent ~= intNew then -- Update progress once per percent or per second, whichever is smaller
intPercent = math.max( 0.1, intNew ) -- Ensure percentage is greater than zero
if intVal > intMax then intVal = intMax intPercent = 100 end -- Ensure values do not exceed maximum
intNew = os.difftime(os.time(),intStart)
if intDelta < intNew then -- Update clock of elapsed time
intDelta = intNew
intScale = math.ceil( intDelta / intPercent ) -- Scale of seconds per percentage step
local intHour = math.floor( intDelta / 3600 )
local intMins = math.floor( intDelta / 60 - intHour * 60 )
local intSecs = intDelta - intMins * 60 - intHour * 3600
strClock = string.format("%02d : %02d : %02d",intHour,intMins,intSecs)
end
doUpdate() -- Update the Progress Bar display
end
iup.LoopStep()
end
end -- function Step
function fh.Focus() -- Bring the Progress Bar window to front
if dlgGauge then doFocus() end
end -- function Focus
function fh.Reset() -- Reset the Progress Bar display
if dlgGauge then doReset() end
end -- function Reset
function fh.Stop() -- Check if Stop button pressed
iup.LoopStep()
return isBarStop
end -- function Stop
function fh.Close() -- Close the Progress Bar window
isBarStop = false
if dlgGauge then dlgGauge:destroy() dlgGauge = nil end
end -- function Close
function fh.Setup(tblSetup) -- Setup optional table of external attributes
if tblSetup then
tblBars = tblSetup
strBack = tblBars.Back or strBack -- Background colour
strBody = tblBars.Body or strBody -- Body text colour
strFont = tblBars.Font or strFont -- Font dialogue
strStop = tblBars.Stop or strStop -- Stop button colour
intPosX = tblBars.X or intPosX -- Window position
intPosY = tblBars.Y or intPosY
end
end -- function Setup
return fh
end -- local function progbar_v3
local progbar = progbar_v3() -- To access FH progress bars module
--[[
@Function: CheckVersionInStore
@Author: Mike Tate
@Version: 1.4
@LastUpdated: 15 Feb 2026
@Description: Check plugin version against version in Plugin Store
@Parameter: Plugin name and version
@Returns: None
@Requires: luacom
@V1.4: Dispense with files and assume called via IUP button;
@V1.3: Save and retrieve latest version in file;
@V1.2: Ensure the Plugin Data folder exists;
@V1.1: Monthly interval between checks; Report if Internet is inaccessible;
@V1.0: Initial version;
]]
function CheckVersionInStore(strPlugin,strVersion) -- Check if later Version available in Plugin Store
require("luacom")
local FSO = luacom.CreateObject("Scripting.FileSystemObject")
local strFile = fhGetContextInfo("CI_APP_DATA_FOLDER").."\\Plugin Data\\VersionInStore "..strPlugin..".dat"
if FSO:FileExists(strFile) then FSO:DeleteFile(strFile,true) end -- Delete obsolete file
local function httpRequest(strRequest) -- Luacom http request protected by pcall() below
local http = luacom.CreateObject("winhttp.winhttprequest.5.1")
http:Open("GET",strRequest,false)
http:Send()
return http.Responsebody
end -- local function httpRequest
local function intVersion(strVersion) -- Convert version string to comparable integer
local intVersion = 0
local arrNumbers = {}
strVersion:gsub("(%d+)", function(strDigits) table.insert(arrNumbers,strDigits) end)
for i = 1, 5 do
intVersion = intVersion * 100 + tonumber(arrNumbers[i] or 0)
end
return intVersion
end -- local function intVersion
local strLatest = "0"
if strPlugin then
local strRequest ="http://www.family-historian.co.uk/lnk/checkpluginversion.php?name="..tostring(strPlugin)
local isOK, strReturn = pcall(httpRequest,strRequest)
if not isOK then -- Problem with Internet access
fhMessageBox(strReturn.."\n The Internet appears to be inaccessible. ")
elseif strReturn then
strLatest = strReturn:match("([%d%.]*),%d*") -- Version digits & dots then comma and Id digits
end
end
local strMessage = "No later Version"
if intVersion(strLatest) > intVersion(strVersion or "0") then
strMessage = "Later Version "..strLatest
end
fhMessageBox(strMessage.." of this Plugin is available from the 'Plugin Store'.")
end -- function CheckVersionInStore
function getParamEmulator() -- Prototype for iup.GetParam(...) -- V1.4
if fhGetAppVersion() > 6 then unpack = table.unpack end
local iupDialog = nil
local arrValues = {}
local isSuccess = false
local strPlugin = fhGetContextInfo("CI_PLUGIN_NAME"):gsub(" %- .*","")
local function handleButton(iupDialog,intIndex,strTitle) -- Handle the dialog buttons
if intIndex == (iup.GETPARAM_OK or -1) then
-- strTitle sometimes needed to determine the function -- 1st button action -- FH V5 needs -1
isSuccess = true
elseif intIndex == (iup.GETPARAM_CANCEL or -3) then -- 2nd 'Cancel Plugin' button -- FH V5 needs -3
isSuccess = false
elseif intIndex == (iup.GETPARAM_HELP or -4) then -- 3rd 'Later Version?' button -- FH V5 needs -4
iupDialog.Active = "NO"
CheckVersionInStore(strPlugin,strVersion)
iupDialog.Active = "YES"
iupDialog.BringFront = "YES"
return 0
end
return 1
end -- function handleButton
local function makeDialog(strTitle,strFormat) -- Make emulated iup.GetParam(...) dialog
local arrFormat = {}
for strForm in strFormat:gmatch(".-\n") do -- Construct parameters from format
local iupParam = iup.param{ format=strForm; }
table.insert(arrFormat,iupParam)
end
local iupParams = iup.parambox{ unpack(arrFormat) }
local iupButton = iup.button{ Title="Help && Advice"; Padding="12x8"; } -- Example of extra button
-- iupDialog = iup.dialog{ Title=strTitle; iup.vbox{ iupParams; iupButton; ALIGNMENT="ACENTER"; MARGIN="10x10"; }; close_cb=function() isSuccess = false return iup.CLOSE end; }
iupDialog = iup.dialog{ Title=strTitle; iupParams; close_cb=function() isSuccess = false return iup.CLOSE end; }
if fhGetAppVersion() > 6 then -- Window centres on FH parent
iup.SetAttribute(iupDialog,"NATIVEPARENT",fhGetContextInfo("CI_PARENT_HWND"))
end
for intParam = 1, iupParams.ParamCount do -- Set all parameter values
local iupParam = iupParams:GetParamParam(intParam-1)
local iupCntrl = iupParam.Control
local anyValue = arrValues[intParam]
if iupParam.Type == "LIST" then anyValue = anyValue + 1 end -- Droplists need an adjustment
iupCntrl.Value = anyValue
end
function iupParams:param_cb(intIndex) -- Parameter call back actions
if intIndex >= 0 then
local iupParam = iupParams:GetParamParam(intIndex) -- Save any parameter value
arrValues[intIndex+1] = tonumber(iupParam.Value) or iupParam.Value
return 1
else
return handleButton(iupDialog,intIndex,strTitle) -- Handle buttons
end
end -- function iupParams:param_cb
function iupButton:action(intButton) -- Display Help Page
local strPlugin = strPlugin:gsub(" ","-"):lower()
fhShellExecute("https://pluginstore.family-historian.co.uk/page/help/"..strPlugin,"","","open")
fhSleep(3000,500)
iupDialog.BringFront = "YES"
return 1
end -- function iupButton:action
end -- local function makeDialog
local function getParam(strTitle,strSize,strFormat,...) -- Emulate iup.GetParam(...)
arrValues = {...}
if fhGetAppVersion() > 6 then
makeDialog(strTitle,strFormat)
if strSize then iupDialog.Size = strSize end
iupDialog:map()
iupDialog.MinSize = iupDialog.NaturalSize
iupDialog:showxy(iup.CENTERPARENT,iup.CENTERPARENT)
if iup.MainLoopLevel()==0 then iup.MainLoop() end
iup.Destroy(iupDialog)
iupDialog = nil
else
local function fncAction(iupDialog,intIndex)
return handleButton(iupDialog,intIndex,strTitle) -- Handle buttons
end -- local function fncAction
arrValues = { iup.GetParam(strTitle,fncAction,strFormat,unpack(arrValues)) }
isSuccess = arrValues[1]
table.remove(arrValues,1)
end
return isSuccess, unpack(arrValues)
end -- local function getParam
return getParam
end -- function getParamEmulator
local getParam = getParamEmulator()
-- Load Data Refs
function loadRefs(strFileName)
local strRefs =
[[
This initial list is popular for cleaning exports to Ancestry websites
Individual Records
%INDI.Fact.AGE% Fact Age
%INDI.Fact.SOUR.PAGE% Fact Citation Where within Text
%INDI.Fact.SOUR.DATA.TEXT% Fact Citation Text From Source
%INDI.Fact.SOUR.NOTE2% Fact Citation local Note
%INDI.SOUR% Whole record Source Citation
%INDI.SOUR2% Whole record Source Note
%INDI.FAMC.PEDI% Family as Child Pedigree (PEDI & _PEDI)
Family Records
%FAM.Fact.AGE% Fact Age
%FAM.Fact.HUSB% Fact Husband's Age
%FAM.Fact.WIFE% Fact Wife's Age
%FAM.Fact.SOUR.PAGE% Fact Citation Where within Text
%FAM.Fact.SOUR.DATA.TEXT% Fact Citation Text From Source
%FAM.Fact.SOUR.NOTE2% Fact Citation local Note
%FAM.SOUR% Whole record Source Citation
%FAM.SOUR2% Whole record Source Note
Source Records
%SOUR._TYPE% Source Type
%SOUR.TEXT% Text From Source
%SOUR.NOTE2% Local Note
Excluded Examples
-%INDI.SOUR[2]% Whole record 2nd Source Citation
-%SOUR.AUTH% Source Author
%REPO% Repository Record
]]
strFileName = strFileName or "?"
if FlgFileExists(strFileName) then -- Read the file in table lines
local strFile = StrLoadFromFile(strFileName)
local dicOption = {}
for strLine in strFile:gmatch("[^\r\n]+") do -- V1.4
local arrFields = split(strLine,"=")
dicOption[arrFields[1]] = arrFields[2]
end
strRefs = dicOption["DataRefs"]:gsub("\01","\n"):gsub("\02","=")-- Reveal Data Refs newlines and =
end
return strRefs
end -- function loadRefs
-- Save the Data Refs to File Name
function saveRefs(strRefs,strFileName)
strFileName = strFileName or "?"
local tblData = {}
local dicOption = {}
dicOption["DataRefs"] = strRefs:gsub("\n","\01"):gsub("=","\02") -- Hide the Data Refs newlines and =
for strField, strValue in pairs(dicOption) do -- Write the file in table lines -- V1.4
table.insert(tblData,strField.."="..strValue.."\n")
end
local strData = table.concat(tblData,"\n")
if not SaveStringToFile(strData,strFileName) then
doError("\nSettings file not saved successfully.\n\nMay need to Delete the following Plugin Data .dat file:\n\n"..strFileName.."\n\nError detected.")
end
end -- function saveRefs
-- Validate one Data Ref
function validRef(strRef) -- V1.2
if not fhIsValidDataRef(strRef) then return false end -- Invalid data ref ~ fhIsValidDataRef(...) does not check [index] is valid
for strInd in strRef:gmatch("%[(.-)%]") do
if not strInd:match("^[1-9]%d-$") then return false end -- Invalid [index]
end
return true
end -- function validRef
-- Check the Data Refs
function checkRefs(strRefs)
for intLine, strLine in ipairs (split(strRefs,"\n")) do
if strLine:match("^%%") then -- Line starts with % so must be validated
local strRef = strLine:gsub("%.Fact",".CENS"):match("^(%%[^%%]+%%)") or ""
if not validRef(strRef) then -- Validate ".Fact" replaced by ".CENS" and extracted leading "%DataRef%"
strLine = strLine:gsub("%%","%%%%")
return false, " Error on line "..intLine.." : "..strLine:gsub("\t"," ")
end
end
end
return true
end -- function checkRefs
-- Check for Project and Warn if in Project Mode
function checkMode()
local strFile = ""
if fhGetContextInfo("CI_APP_MODE") == "Project Mode" then
local strAns = fhMessageBox(
[[
Warning:
This plugin is designed to delete data from the current file.
You have a Project open.
Please confirm you want to clean the data from
]]..fhGetContextInfo("CI_PROJECT_NAME"),
"MB_OKCANCEL",
"MB_ICONEXCLAMATION")
if strAns ~= "OK" then
return false
end
strFile = fhGetPluginDataFileName() -- Use Project sticky data refs -- V1.2
else
strFile = fhGetPluginDataFileName("LOCAL_MACHINE") -- Use global sticky data refs needs V5.0.8 -- V1.2
end
return strFile:gsub(" %- V%d.*%.dat$",".dat") -- All versions use same Plugin Data file -- V1.2
end -- function checkMode
-- User Dialogue
local tblDataCounts = {} -- Table of data reference usage counts
local arrDeleteList = {} -- Array of data pointers to delete
function getRefs()
local fileRefs = checkMode()
if not fileRefs then return nil end
local strRefs = loadRefs(fileRefs)
local strHelp = " Use Shft+Tab to enter tab char. Drag edge of dialogue to adjust window size."
local strErrs = strHelp -- V1.2
repeat
strRefs, intLines = strRefs:gsub("\n","\n") -- Count number of lines
local strParam =
" Edit the list below with a full %%Data Reference%% for each field to be deleted. \r"..
" Use the pseudo tag 'Fact' to represent all of your Event and Attribute facts. \r"..
" Each entry must be on a separate line, and enclosed in %% percentage characters. \r"..
" Notes added to any entry after the second %% percentage character are ignored. \r"..
" Any line NOT having a %% percentage character against left margin is ignored. \r"..
" Any invalid %%Data Reference%% is reported, and unused ones listed afterwards. \r\r"..
" Data References: %m\n"..
" To restore default settings, erase 'Data References:' and 'Apply Data References' \r".. -- V1.2
" "..strErrs.." %t\n"..
" Button Names %u[ Apply Data References , Cancel Plugin , Later Version? ]\n" -- V1.4
if fhGetAppVersion() == 5 then
strParam = strParam:gsub("%%%%","º/o")
end
local _,intLines = strRefs:gsub("\n","\n") -- Adjusts size height according to lines of refs -- V1.4
local intHigh = math.min( intLines * 8 + 160, 450 )
local strSize = "x"..tostring(intHigh)
local theAns, strAns = getParam(strPluginName,strSize,strParam,strRefs) -- V1.4
if not theAns then
saveRefs(strRefs,fileRefs) -- Save original Data Refs and current dialogue position
return nil
end
if #strAns < 4 then -- If empty Data Refs then restore defaults
strRefs = loadRefs()
saveRefs(strRefs,fileRefs) -- Save reset Data Refs and current dialogue position
strErrs = strHelp -- V1.2
theAns = false
else
strRefs = strAns
saveRefs(strRefs,fileRefs) -- Save Data Refs and current dialogue position -- V1.2
theAns, strErrs = checkRefs(strRefs) -- Check Data Refs format
end
until theAns == true
saveRefs(strRefs,fileRefs) -- Save latest Data Refs and current dialogue position
return strRefs
end -- function getRefs
-- Make the Result Set arrays
local arrNum = {} -- Numerical order
local arrRec = {} -- Parent Record of Deleted Item
local arrBud = {} -- Parent Record buddy pointer
local arrRid = {} -- Parent Record Id
local arrDel = {} -- Text of Deleted Item
local arrPar = {} -- Parent Item of Deleted Item
local arrRef = {} -- Data Reference of Deleted Item
function makeResultSet(ptrDel,strRef) -- Compile the Result Set for Deleted Item matching Data Ref -- V1.2
local intCount = tblDataCounts[strRef]
if intCount <= 9 then -- Only list first 9 deletions per Data Ref
local ptrRec = fhNewItemPtr()
ptrRec:MoveToRecordItem(ptrDel) -- Get the Parent Record of Deleted Item
local strDel = ""
if ptrRec:IsSame(ptrDel) then strDel = " (deleted)" end -- Identify deleted record
table.insert( arrNum,(arrNum[#arrNum] or 0) + 1 ) -- Numerical order of entries
table.insert( arrRec,fhGetDisplayText(ptrRec,"","min")..strDel )-- Parent Record text
table.insert( arrBud,ptrRec:Clone() ) -- Parent Record buddy pointer
table.insert( arrRid,fhGetRecordId(ptrRec) ) -- Parent Record Id
table.insert( arrDel,fhGetDisplayText(ptrDel,"","min") ) -- Deleted Item text
ptrDel:MoveToParentItem(ptrDel)
table.insert( arrPar,ptrDel:Clone() ) -- Parent Item pointer
table.insert( arrRef,"%"..strRef.."%" ) -- Data Reference text
arrRef[strRef] = #arrRef
else
intCount = intCount - 9 -- Identify how many more like this
arrRef[arrRef[strRef]] = "%"..strRef.."% and "..intCount.." more like this..."
end
end -- function makeResultSet
-- Build the Data Tag and its Index
function buildIndex(dicRef,ptrTag) -- Cater for multiple tag instance index -- V1.2
local strTag = fhGetTag(ptrTag)
local intTag = ( dicRef[strTag] or 0 ) + 1 -- Count each instance of same tag
dicRef[strTag] = intTag
return strTag, "["..intTag.."]" -- Return tag name and its index
end -- function buildIndex
-- Process Fact or Tag
function processTag(ptrTag,dicTags)
if dicTags then -- Ignore if tag not listed
if #dicTags == 1 then
table.insert(arrDeleteList,ptrTag:Clone()) -- Delete final leaf tag
local strRef = dicTags[1]
tblDataCounts[strRef] = tblDataCounts[strRef] + 1 -- Count deletions
makeResultSet(ptrTag,strRef) -- V1.2
else
local dicRef = {} -- List counting multiple tag instances -- V1.2
ptrTag:MoveToFirstChildItem(ptrTag)
while ptrTag:IsNotNull() do
local strTag, strInd = buildIndex(dicRef,ptrTag) -- Cater for multiple tag instance index -- V1.2
processTag(ptrTag:Clone(),dicTags[strTag]) -- Search listed child tags without index -- V1.2
processTag(ptrTag:Clone(),dicTags[strTag..strInd]) -- Search listed child tags with an index -- V1.2
ptrTag:MoveNext()
end
end
end
end -- function processTag
-- Process File
function Main()
local strRefs = getRefs()
if not strRefs then return end
local dicType = {} -- Dictionary of nested data ref tags
for _, strRef in ipairs (split(strRefs,"\n")) do
strRef = strRef:match("^%%([^%%]+)%%")
if strRef then -- Found %data ref%
local dicType = dicType
local arrTag = split(strRef,".")
for intTag, strTag in ipairs (arrTag) do -- Convert data ref into nested dictionary table
if not dicType[strTag] then
if intTag == 1 then table.insert(dicType,strTag) end -- Note the record tag order -- V1.2
dicType[strTag] = {} -- Create next nested table
end
dicType = dicType[strTag]
end
table.insert(dicType,strRef) -- Save final leaf tag data ref
table.insert(tblDataCounts,strRef)
tblDataCounts[strRef] = 0 -- Zero data ref deletion count
end
end
local intRec = 0 -- Is progbar needed? -- V1.4
for _, strType in ipairs (dicType) do
local strRec = strType:match("^([_%u]+)") -- Extract record tag name
local ptrRec = fhNewItemPtr()
ptrRec:MoveToFirstRecord(strRec)
while ptrRec:IsNotNull() do -- Count each record type -- V1.4
intRec = intRec + 1
ptrRec:MoveNext()
end
end
if intRec > 10000 then progbar.Start("Finding Unwanted Fields",intRec) end -- V1.4
intRec = 0
for _, strType in ipairs (dicType) do -- Search all matching data refs in record tag order -- V1.2
local dicTags = dicType[strType]
local strRec = strType:match("^([_%u]+)") -- Extract record tag name -- V1.2
local intRid = tonumber(strType:match("%[(%d+)%]$")) -- Extract record id if any -- V1.2
local ptrRec = fhNewItemPtr()
ptrRec:MoveToFirstRecord(strRec)
while ptrRec:IsNotNull() do -- Search each record type
progbar.Step(1) -- V1.4
if progbar.Stop() then break end
if not intRid or intRid == fhGetRecordId(ptrRec) then -- with matching Record Id -- V1.2
if #dicTags == 1 then
table.insert(arrDeleteList,ptrRec:Clone()) -- Delete record and count
tblDataCounts[strType] = tblDataCounts[strType] + 1
makeResultSet(ptrRec,strType) -- V1.2
else
local ptrTag = fhNewItemPtr()
local dicRef = {} -- List counting multiple tag instances -- V1.2
ptrTag:MoveToFirstChildItem(ptrRec)
while ptrTag:IsNotNull() do
local strTag, strInd = buildIndex(dicRef,ptrTag) -- Cater for multiple tag instance index -- V1.2
processTag(ptrTag:Clone(),dicTags[strTag]) -- Search specific tags without index -- V1.2
processTag(ptrTag:Clone(),dicTags[strTag..strInd])-- Search specific tags with an index -- V1.2
if fhIsFact(ptrTag) then
processTag(ptrTag:Clone(),dicTags["Fact"]) -- Search generic Facts without index -- V1.2
processTag(ptrTag:Clone(),dicTags["Fact"..strInd])-- Search generic Facts with an index -- V1.2
end
ptrTag:MoveNext()
end
end
if intRid then break end -- Escape if single Record Id defined -- V1.2
end
ptrRec:MoveNext()
end
end
progbar.Close() -- V1.4
local strZero = ""
for _, strRef in ipairs (tblDataCounts) do -- Find unused/invalid data refs
if tblDataCounts[strRef] == 0 then
strZero = strZero.."%"..strRef.."%\n"
end
end
if #strZero > 0 then
strZero = "\n\nThese data refs were not used:\n"..strZero
end
local strAns = fhMessageBox("Please Confirm Changes:\n\n"..#arrDeleteList.." items will be deleted.\n\nTo reverse those deletions use:\nEdit > Undo Plugin Updates"..strZero,"MB_OKCANCEL")
if strAns == "OK" then
for _, ptrItem in ipairs(arrDeleteList) do
fhDeleteItem(ptrItem)
end
if #arrRec > 0 then -- Output the Result Set details -- V1.2
fhOutputResultSetTitles(strPluginName)
fhOutputResultSetColumn("No." ,"integer",arrNum,#arrRec, 20)
fhOutputResultSetColumn("Parent Record" ,"text",arrRec,#arrRec,200)
fhOutputResultSetColumn("Parent Record" ,"item",arrBud,#arrRec,200,"align_left",0,true,"default","buddy")
fhOutputResultSetColumn("RecId" ,"integer",arrRid,#arrRec, 25)
fhOutputResultSetColumn("Data Reference","text",arrRef,#arrRec,200)
fhOutputResultSetColumn("Parent Item" ,"item",arrPar,#arrRec,200)
fhOutputResultSetColumn("Deleted Item" ,"text",arrDel,#arrRec,400)
end
end
end -- function Main
fhInitialise(5,0,8,"save_recommended") -- V5.0.8 for sticky settings with scope
Main()
--[[
@Title: Clean Unwanted Fields
@Type: Standard
@Author: Mike Tate
@Contributors:
@Version: 1.4
@Keywords:
@LastUpdated: 16 Feb 2026
@Licence: This plugin is copyright (c) 2026 Mike Tate & contributors and is licensed under the MIT License which is hereby incorporated by reference (see https://pluginstore.family-historian.co.uk/fh-plugin-licence)
@Description: Removes fields that typically upset Ancestry, or hold sensitive data, or any chosen by user.
It is meant to be used in conjunction with the Export GEDCOM File and Split Tree Helper commands,
so will prompt for confirmation if you try to use it on a Gedcom which is open in Project mode.
@V1.4: Centre windows on FH window; Check for Updates button; Ch-ViS 1.4; progbar 3.1; FSO filename accents;
@V1.3: FH V7 Lua 3.5 IUP 3.28 compatible;
@V1.2: Cater for data refs with instances such as %INDI.NAME[2]%, add a Result Set, ensure all versions use one Plugin Data .dat file.
@V1.1: Now remembers its screen position, handles specific Fact tags, and more rigorous data ref validation check.
@V1.0: First publication in the Plugin Store.
@V0.5: Add user dialogue and sticky Data Reference list.
@V0.4: ?
@V0.3: Third prototype with remove whole record example.
@V0.2: Second prototype redesigned and renamed to allow user edits.
@V0.1: First prototype was Clean Ancestry Fields.
]]
require("iuplua") -- To access IUP GUI library
require("luacom") -- To create File System Object -- V1.4
local FSO = luacom.CreateObject("Scripting.FileSystemObject")
local strVersion = "1.4"
local strPluginName = "Clean Unwanted Fields "..strVersion -- Update title and version number here
if fhGetAppVersion() > 5 then -- Cater for Unicode UTF-8 from FH Version 6 onwards
fhSetStringEncoding("UTF-8")
iup.SetGlobal("UTF8MODE","YES")
iup.SetGlobal("CUSTOMQUITMESSAGE","YES") -- Needed for IUP 3.28 -- V1.4
end
function Encoding()
if fhGetAppVersion() > 5 then return fhGetStringEncoding() end
return "ANSI"
end -- function Encoding
-- Split a string using "," or chosen separator --
function split(strTxt,strSep)
local tblFields = {}
local strPattern = string.format("([^%s]+)", strSep or ",")
strTxt = tostring(strTxt or "")
strTxt:gsub(strPattern, function(strField) tblFields[#tblFields+1] = strField end)
return tblFields
end -- function split
-- Report error message --
local function doError(strMessage,errFunction)
-- strMessage ~ error message text
-- errFunction ~ optional error reporting function
if type(errFunction) == "function" then
errFunction(strMessage)
else
error(strMessage)
end
end -- function doError
-- Convert filename to ANSI alternative and indicate success --
function FileNameToANSI(strFileName,strAnsiName)
-- strFileName ~ full file path
-- strAnsiFile ~ ANSI file name & type
-- return values ~ ANSI file path, true if original path was ANSI compatible
if Encoding() == "ANSI" then return strFileName, true end
local isFlag = fhIsConversionLossFlagSet()
fhSetConversionLossFlag(false)
local strAnsi = fhConvertUTF8toANSI(strFileName)
local wasAnsi = true
if fhIsConversionLossFlagSet() then
strAnsiName = strAnsiName or "ANSI.ANSI"
strAnsi = fhGetContextInfo("CI_APP_DATA_FOLDER").."\\Plugin Data\\"..strAnsiName
wasAnsi = false
end
fhSetConversionLossFlag(isFlag)
return strAnsi, wasAnsi
end -- function FileNameToANSI
-- Get parent folder --
function GetParentFolder(strFileName)
-- strFileName ~ full file path
-- return value ~ parent folder path
local strParent = FSO:GetParentFolderName(strFileName) --! Faulty in FH v6 with Unicode chars in path
if fhGetAppVersion() == 6 then
local _, wasAnsi = FileNameToANSI(strFileName)
if not wasAnsi then
strParent = strFileName:match("^(.+)[\\/][^\\/]+[\\/]?$")
end
end
return strParent
end -- function GetParentFolder
-- Check if file exists --
function FlgFileExists(strFileName)
-- strFileName ~ full file path
-- return value ~ true if it exists
return FSO:FileExists(strFileName)
end -- function FlgFileExists
-- Delete a file if it exists --
function DeleteFile(strFileName,errFunction)
-- strFileName ~ full file path
-- errFunction ~ optional error reporting function
-- return value ~ true if file does not exist or is deleted else false
if FSO:FileExists(strFileName) then
FSO:DeleteFile(strFileName,true)
if FSO:FileExists(strFileName) then
doError("File Not Deleted:\n"..strFileName.."\n",errFunction)
return false
end
end
return true
end -- function DeleteFile
-- Copy a file if it exists and destination is not a folder --
function CopyFile(strFileName,strDestination)
-- strFileName ~ full source file path
-- strDestination~ full target file path
-- return value ~ true if file exists and is copied else false
if MakeFolder(GetParentFolder(strDestination)) and FSO:FileExists(strFileName) and not FSO:FolderExists(strDestination) then
FSO:CopyFile(strFileName,strDestination)
if FSO:FileExists(strDestination) then
return true
end
end
return false
end -- function CopyFile
-- Move a file if it exists and destination is not a folder --
function MoveFile(strFileName,strDestination)
-- strFileName ~ full source file path
-- strDestination~ full target file path
-- return value ~ true if file exists and is moved else false
if MakeFolder(GetParentFolder(strDestination)) and FSO:FileExists(strFileName) and not FSO:FolderExists(strDestination) then
if DeleteFile(strDestination) then
FSO:MoveFile(strFileName,strDestination)
if FSO:FileExists(strDestination) then
return true
end
end
end
return false
end -- function MoveFile
-- Open File with ANSI path and return Handle --
function OpenFile(strFileName,strMode)
-- strFileName ~ full file path
-- strMode ~ "r", "w", "a" optionally suffixed with "+" &/or "b"
-- return value ~ file handle
local fileHandle, strError = io.open(strFileName,strMode)
if fileHandle == nil then
error("\n Unable to open file in \""..strMode.."\" mode. \n "..strFileName.." \n "..strError.." \n")
end
return fileHandle
end -- function OpenFile
-- Save string to file --
function SaveStringToFile(strContents,strFileName,strFormat)
-- strContents ~ text string
-- strFileName ~ full file path
-- strFormat ~ optional "UTF-8" or "UTF-16LE"
-- return value ~ true if successful else false
strFormat = strFormat or "UTF-8"
if fhGetAppVersion() > 6 then
return fhSaveTextFile(strFileName,strContents,strFormat)
end
local strAnsi, wasAnsi = FileNameToANSI(strFileName)
local fileHandle = OpenFile(strAnsi,"w")
fileHandle:write(strContents)
assert(fileHandle:close())
if not wasAnsi then
MoveFile(strAnsi,strFileName)
end
return true
end -- function SaveStringToFile
-- Load string from file --
function StrLoadFromFile(strFileName,strFormat)
-- strFileName ~ full file path
-- strFormat ~ optional "UTF-8" or "UTF-16LE"
-- return value ~ file contents
strFormat = strFormat or "UTF-8"
if fhGetAppVersion() > 6 then
return fhLoadTextFile(strFileName,strFormat)
end
local strAnsi, wasAnsi = FileNameToANSI(strFileName)
if not wasAnsi then
CopyFile(strFileName,strAnsi)
end
local fileHandle = OpenFile(strAnsi,"r")
local strContents = fileHandle:read("*all")
assert(fileHandle:close())
return strContents
end -- function StrLoadFromFile
--[[
@Module: +fh+progbar_v3
@Author: Mike Tate
@Version: 3.1
@LastUpdated: 23 Jan 2026
@Description: Progress Bar library module.
@V3.1: Use NATIVEPARENT amd CENTERPARENT.
@V3.0: Function Prototype Closure version.
@V1.0: Initial version.
]]
local function progbar_v3()
local fh = {} -- Local environment table
require "iuplua" -- To access GUI window builder
iup.SetGlobal("CUSTOMQUITMESSAGE","YES") -- Needed for IUP 3.28
local tblBars = {} -- Table for optional external attributes
local strBack = "255 255 255" -- Background colour default is white
local strBody = "0 0 0" -- Body text colour default is black
local strFont = nil -- Font dialogue default is current font
local strStop = "255 0 0" -- Stop button colour default is red
local intPosX = iup.CENTERPARENT -- Show window default position is central -- V3.1
local intPosY = iup.CENTERPARENT
local intMax, intVal, intPercent, intStart, intDelta, intScale, strClock, isBarStop
local lblText, barGauge, lblDelta, btnStop, dlgGauge
local function doFocus() -- Bring the Progress Bar window into Focus
dlgGauge.BringFront="YES" -- If used too often, inhibits other windows scroll bars, etc
end -- local function doFocus
local function doUpdate() -- Update the Progress Gauge and the Delta % with clock
barGauge.Value = intVal
lblDelta.Title = string.format("%4d %% %s ",math.floor(intPercent),strClock)
end -- local function doUpdate
local function doReset() -- Reset all dialogue variables and Update display
intVal = 0 -- Current value of Progress Bar
intPercent= 0.01 -- Percentage of progress
intStart = os.time() -- Start time of progress
intDelta = 0 -- Delta time of progress
intScale = math.ceil( intMax / 1000 ) -- Scale of percentage per second of progress (initial guess is corrected in Step function)
strClock = "00 : 00 : 00" -- Clock delta time display
isBarStop = false -- Stop button pressed signal
doUpdate()
doFocus()
end -- local function doReset
function fh.Start(strTitle,intMaximum) -- Create & start Progress Bar window
if not dlgGauge then
strTitle = strTitle or "" -- Dialogue and button title
intMax = intMaximum or 100 -- Maximun range of Progress Bar, default is 100
local strSize = tostring( math.max( 100, string.len(" Stop "..strTitle) * 8 ) ).."x30" -- Adjust Stop button size to Title
lblText = iup.label { Title=" "; Expand="YES"; Alignment="ACENTER"; Tip="Progress Message"; }
barGauge = iup.progressbar { RasterSize="400x30"; Value=0; Max=intMax; Tip="Progress Bar"; }
lblDelta = iup.label { Title=" "; Expand="YES"; Alignment="ACENTER"; Tip="Percentage and Elapsed Time"; }
btnStop = iup.button { Title=" Stop "..strTitle; RasterSize=strSize; FgColor=strStop; Tip="Stop Progress Button"; action=function() isBarStop = true end; } -- Signal Stop button pressed return iup.CLOSE -- Often caused main GUI to close !!!
dlgGauge = iup.dialog { Title=strTitle.." Progress "; Font=strFont; FgColor=strBody; Background=strBack; DialogFrame="YES"; -- Remove Windows minimize/maximize menu
iup.vbox{ Alignment="ACENTER"; Gap="10"; Margin="10x10";
lblText;
barGauge;
lblDelta;
btnStop;
};
move_cb = function(self,x,y) tblBars.X = x tblBars.Y = y end;
close_cb = btnStop.action; -- Windows Close button = Stop button
}
if type(tblBars.GUI) == "table"
and type(tblBars.GUI.ShowDialogue) == "function" then
dlgGauge.move_cb = nil -- Use GUI library to show & move window
tblBars.GUI.ShowDialogue("Bars",dlgGauge,btnStop,"showxy")
else
if fhGetAppVersion() > 6 then -- Window centres on FH parent -- V3.1
iup.SetAttribute(dlgGauge,"NATIVEPARENT",fhGetContextInfo("CI_PARENT_HWND"))
end
dlgGauge:showxy(intPosX,intPosY) -- Show the Progress Bar window
end
doReset() -- Reset the Progress Bar display
end
end -- function Start
function fh.Message(strText) -- Show the Progress Bar message
if dlgGauge then lblText.Title = strText end
end -- function Message
function fh.Step(intStep) -- Step the Progress Bar forward
if dlgGauge then
intVal = intVal + ( intStep or 1 ) -- Default step is 1
local intNew = math.ceil( intVal / intMax * 100 * intScale ) / intScale
if intPercent ~= intNew then -- Update progress once per percent or per second, whichever is smaller
intPercent = math.max( 0.1, intNew ) -- Ensure percentage is greater than zero
if intVal > intMax then intVal = intMax intPercent = 100 end -- Ensure values do not exceed maximum
intNew = os.difftime(os.time(),intStart)
if intDelta < intNew then -- Update clock of elapsed time
intDelta = intNew
intScale = math.ceil( intDelta / intPercent ) -- Scale of seconds per percentage step
local intHour = math.floor( intDelta / 3600 )
local intMins = math.floor( intDelta / 60 - intHour * 60 )
local intSecs = intDelta - intMins * 60 - intHour * 3600
strClock = string.format("%02d : %02d : %02d",intHour,intMins,intSecs)
end
doUpdate() -- Update the Progress Bar display
end
iup.LoopStep()
end
end -- function Step
function fh.Focus() -- Bring the Progress Bar window to front
if dlgGauge then doFocus() end
end -- function Focus
function fh.Reset() -- Reset the Progress Bar display
if dlgGauge then doReset() end
end -- function Reset
function fh.Stop() -- Check if Stop button pressed
iup.LoopStep()
return isBarStop
end -- function Stop
function fh.Close() -- Close the Progress Bar window
isBarStop = false
if dlgGauge then dlgGauge:destroy() dlgGauge = nil end
end -- function Close
function fh.Setup(tblSetup) -- Setup optional table of external attributes
if tblSetup then
tblBars = tblSetup
strBack = tblBars.Back or strBack -- Background colour
strBody = tblBars.Body or strBody -- Body text colour
strFont = tblBars.Font or strFont -- Font dialogue
strStop = tblBars.Stop or strStop -- Stop button colour
intPosX = tblBars.X or intPosX -- Window position
intPosY = tblBars.Y or intPosY
end
end -- function Setup
return fh
end -- local function progbar_v3
local progbar = progbar_v3() -- To access FH progress bars module
--[[
@Function: CheckVersionInStore
@Author: Mike Tate
@Version: 1.4
@LastUpdated: 15 Feb 2026
@Description: Check plugin version against version in Plugin Store
@Parameter: Plugin name and version
@Returns: None
@Requires: luacom
@V1.4: Dispense with files and assume called via IUP button;
@V1.3: Save and retrieve latest version in file;
@V1.2: Ensure the Plugin Data folder exists;
@V1.1: Monthly interval between checks; Report if Internet is inaccessible;
@V1.0: Initial version;
]]
function CheckVersionInStore(strPlugin,strVersion) -- Check if later Version available in Plugin Store
require("luacom")
local FSO = luacom.CreateObject("Scripting.FileSystemObject")
local strFile = fhGetContextInfo("CI_APP_DATA_FOLDER").."\\Plugin Data\\VersionInStore "..strPlugin..".dat"
if FSO:FileExists(strFile) then FSO:DeleteFile(strFile,true) end -- Delete obsolete file
local function httpRequest(strRequest) -- Luacom http request protected by pcall() below
local http = luacom.CreateObject("winhttp.winhttprequest.5.1")
http:Open("GET",strRequest,false)
http:Send()
return http.Responsebody
end -- local function httpRequest
local function intVersion(strVersion) -- Convert version string to comparable integer
local intVersion = 0
local arrNumbers = {}
strVersion:gsub("(%d+)", function(strDigits) table.insert(arrNumbers,strDigits) end)
for i = 1, 5 do
intVersion = intVersion * 100 + tonumber(arrNumbers[i] or 0)
end
return intVersion
end -- local function intVersion
local strLatest = "0"
if strPlugin then
local strRequest ="http://www.family-historian.co.uk/lnk/checkpluginversion.php?name="..tostring(strPlugin)
local isOK, strReturn = pcall(httpRequest,strRequest)
if not isOK then -- Problem with Internet access
fhMessageBox(strReturn.."\n The Internet appears to be inaccessible. ")
elseif strReturn then
strLatest = strReturn:match("([%d%.]*),%d*") -- Version digits & dots then comma and Id digits
end
end
local strMessage = "No later Version"
if intVersion(strLatest) > intVersion(strVersion or "0") then
strMessage = "Later Version "..strLatest
end
fhMessageBox(strMessage.." of this Plugin is available from the 'Plugin Store'.")
end -- function CheckVersionInStore
function getParamEmulator() -- Prototype for iup.GetParam(...) -- V1.4
if fhGetAppVersion() > 6 then unpack = table.unpack end
local iupDialog = nil
local arrValues = {}
local isSuccess = false
local strPlugin = fhGetContextInfo("CI_PLUGIN_NAME"):gsub(" %- .*","")
local function handleButton(iupDialog,intIndex,strTitle) -- Handle the dialog buttons
if intIndex == (iup.GETPARAM_OK or -1) then
-- strTitle sometimes needed to determine the function -- 1st button action -- FH V5 needs -1
isSuccess = true
elseif intIndex == (iup.GETPARAM_CANCEL or -3) then -- 2nd 'Cancel Plugin' button -- FH V5 needs -3
isSuccess = false
elseif intIndex == (iup.GETPARAM_HELP or -4) then -- 3rd 'Later Version?' button -- FH V5 needs -4
iupDialog.Active = "NO"
CheckVersionInStore(strPlugin,strVersion)
iupDialog.Active = "YES"
iupDialog.BringFront = "YES"
return 0
end
return 1
end -- function handleButton
local function makeDialog(strTitle,strFormat) -- Make emulated iup.GetParam(...) dialog
local arrFormat = {}
for strForm in strFormat:gmatch(".-\n") do -- Construct parameters from format
local iupParam = iup.param{ format=strForm; }
table.insert(arrFormat,iupParam)
end
local iupParams = iup.parambox{ unpack(arrFormat) }
local iupButton = iup.button{ Title="Help && Advice"; Padding="12x8"; } -- Example of extra button
-- iupDialog = iup.dialog{ Title=strTitle; iup.vbox{ iupParams; iupButton; ALIGNMENT="ACENTER"; MARGIN="10x10"; }; close_cb=function() isSuccess = false return iup.CLOSE end; }
iupDialog = iup.dialog{ Title=strTitle; iupParams; close_cb=function() isSuccess = false return iup.CLOSE end; }
if fhGetAppVersion() > 6 then -- Window centres on FH parent
iup.SetAttribute(iupDialog,"NATIVEPARENT",fhGetContextInfo("CI_PARENT_HWND"))
end
for intParam = 1, iupParams.ParamCount do -- Set all parameter values
local iupParam = iupParams:GetParamParam(intParam-1)
local iupCntrl = iupParam.Control
local anyValue = arrValues[intParam]
if iupParam.Type == "LIST" then anyValue = anyValue + 1 end -- Droplists need an adjustment
iupCntrl.Value = anyValue
end
function iupParams:param_cb(intIndex) -- Parameter call back actions
if intIndex >= 0 then
local iupParam = iupParams:GetParamParam(intIndex) -- Save any parameter value
arrValues[intIndex+1] = tonumber(iupParam.Value) or iupParam.Value
return 1
else
return handleButton(iupDialog,intIndex,strTitle) -- Handle buttons
end
end -- function iupParams:param_cb
function iupButton:action(intButton) -- Display Help Page
local strPlugin = strPlugin:gsub(" ","-"):lower()
fhShellExecute("https://pluginstore.family-historian.co.uk/page/help/"..strPlugin,"","","open")
fhSleep(3000,500)
iupDialog.BringFront = "YES"
return 1
end -- function iupButton:action
end -- local function makeDialog
local function getParam(strTitle,strSize,strFormat,...) -- Emulate iup.GetParam(...)
arrValues = {...}
if fhGetAppVersion() > 6 then
makeDialog(strTitle,strFormat)
if strSize then iupDialog.Size = strSize end
iupDialog:map()
iupDialog.MinSize = iupDialog.NaturalSize
iupDialog:showxy(iup.CENTERPARENT,iup.CENTERPARENT)
if iup.MainLoopLevel()==0 then iup.MainLoop() end
iup.Destroy(iupDialog)
iupDialog = nil
else
local function fncAction(iupDialog,intIndex)
return handleButton(iupDialog,intIndex,strTitle) -- Handle buttons
end -- local function fncAction
arrValues = { iup.GetParam(strTitle,fncAction,strFormat,unpack(arrValues)) }
isSuccess = arrValues[1]
table.remove(arrValues,1)
end
return isSuccess, unpack(arrValues)
end -- local function getParam
return getParam
end -- function getParamEmulator
local getParam = getParamEmulator()
-- Load Data Refs
function loadRefs(strFileName)
local strRefs =
[[
This initial list is popular for cleaning exports to Ancestry websites
Individual Records
%INDI.Fact.AGE% Fact Age
%INDI.Fact.SOUR.PAGE% Fact Citation Where within Text
%INDI.Fact.SOUR.DATA.TEXT% Fact Citation Text From Source
%INDI.Fact.SOUR.NOTE2% Fact Citation local Note
%INDI.SOUR% Whole record Source Citation
%INDI.SOUR2% Whole record Source Note
%INDI.FAMC.PEDI% Family as Child Pedigree (PEDI & _PEDI)
Family Records
%FAM.Fact.AGE% Fact Age
%FAM.Fact.HUSB% Fact Husband's Age
%FAM.Fact.WIFE% Fact Wife's Age
%FAM.Fact.SOUR.PAGE% Fact Citation Where within Text
%FAM.Fact.SOUR.DATA.TEXT% Fact Citation Text From Source
%FAM.Fact.SOUR.NOTE2% Fact Citation local Note
%FAM.SOUR% Whole record Source Citation
%FAM.SOUR2% Whole record Source Note
Source Records
%SOUR._TYPE% Source Type
%SOUR.TEXT% Text From Source
%SOUR.NOTE2% Local Note
Excluded Examples
-%INDI.SOUR[2]% Whole record 2nd Source Citation
-%SOUR.AUTH% Source Author
%REPO% Repository Record
]]
strFileName = strFileName or "?"
if FlgFileExists(strFileName) then -- Read the file in table lines
local strFile = StrLoadFromFile(strFileName)
local dicOption = {}
for strLine in strFile:gmatch("[^\r\n]+") do -- V1.4
local arrFields = split(strLine,"=")
dicOption[arrFields[1]] = arrFields[2]
end
strRefs = dicOption["DataRefs"]:gsub("\01","\n"):gsub("\02","=")-- Reveal Data Refs newlines and =
end
return strRefs
end -- function loadRefs
-- Save the Data Refs to File Name
function saveRefs(strRefs,strFileName)
strFileName = strFileName or "?"
local tblData = {}
local dicOption = {}
dicOption["DataRefs"] = strRefs:gsub("\n","\01"):gsub("=","\02") -- Hide the Data Refs newlines and =
for strField, strValue in pairs(dicOption) do -- Write the file in table lines -- V1.4
table.insert(tblData,strField.."="..strValue.."\n")
end
local strData = table.concat(tblData,"\n")
if not SaveStringToFile(strData,strFileName) then
doError("\nSettings file not saved successfully.\n\nMay need to Delete the following Plugin Data .dat file:\n\n"..strFileName.."\n\nError detected.")
end
end -- function saveRefs
-- Validate one Data Ref
function validRef(strRef) -- V1.2
if not fhIsValidDataRef(strRef) then return false end -- Invalid data ref ~ fhIsValidDataRef(...) does not check [index] is valid
for strInd in strRef:gmatch("%[(.-)%]") do
if not strInd:match("^[1-9]%d-$") then return false end -- Invalid [index]
end
return true
end -- function validRef
-- Check the Data Refs
function checkRefs(strRefs)
for intLine, strLine in ipairs (split(strRefs,"\n")) do
if strLine:match("^%%") then -- Line starts with % so must be validated
local strRef = strLine:gsub("%.Fact",".CENS"):match("^(%%[^%%]+%%)") or ""
if not validRef(strRef) then -- Validate ".Fact" replaced by ".CENS" and extracted leading "%DataRef%"
strLine = strLine:gsub("%%","%%%%")
return false, " Error on line "..intLine.." : "..strLine:gsub("\t"," ")
end
end
end
return true
end -- function checkRefs
-- Check for Project and Warn if in Project Mode
function checkMode()
local strFile = ""
if fhGetContextInfo("CI_APP_MODE") == "Project Mode" then
local strAns = fhMessageBox(
[[
Warning:
This plugin is designed to delete data from the current file.
You have a Project open.
Please confirm you want to clean the data from
]]..fhGetContextInfo("CI_PROJECT_NAME"),
"MB_OKCANCEL",
"MB_ICONEXCLAMATION")
if strAns ~= "OK" then
return false
end
strFile = fhGetPluginDataFileName() -- Use Project sticky data refs -- V1.2
else
strFile = fhGetPluginDataFileName("LOCAL_MACHINE") -- Use global sticky data refs needs V5.0.8 -- V1.2
end
return strFile:gsub(" %- V%d.*%.dat$",".dat") -- All versions use same Plugin Data file -- V1.2
end -- function checkMode
-- User Dialogue
local tblDataCounts = {} -- Table of data reference usage counts
local arrDeleteList = {} -- Array of data pointers to delete
function getRefs()
local fileRefs = checkMode()
if not fileRefs then return nil end
local strRefs = loadRefs(fileRefs)
local strHelp = " Use Shft+Tab to enter tab char. Drag edge of dialogue to adjust window size."
local strErrs = strHelp -- V1.2
repeat
strRefs, intLines = strRefs:gsub("\n","\n") -- Count number of lines
local strParam =
" Edit the list below with a full %%Data Reference%% for each field to be deleted. \r"..
" Use the pseudo tag 'Fact' to represent all of your Event and Attribute facts. \r"..
" Each entry must be on a separate line, and enclosed in %% percentage characters. \r"..
" Notes added to any entry after the second %% percentage character are ignored. \r"..
" Any line NOT having a %% percentage character against left margin is ignored. \r"..
" Any invalid %%Data Reference%% is reported, and unused ones listed afterwards. \r\r"..
" Data References: %m\n"..
" To restore default settings, erase 'Data References:' and 'Apply Data References' \r".. -- V1.2
" "..strErrs.." %t\n"..
" Button Names %u[ Apply Data References , Cancel Plugin , Later Version? ]\n" -- V1.4
if fhGetAppVersion() == 5 then
strParam = strParam:gsub("%%%%","º/o")
end
local _,intLines = strRefs:gsub("\n","\n") -- Adjusts size height according to lines of refs -- V1.4
local intHigh = math.min( intLines * 8 + 160, 450 )
local strSize = "x"..tostring(intHigh)
local theAns, strAns = getParam(strPluginName,strSize,strParam,strRefs) -- V1.4
if not theAns then
saveRefs(strRefs,fileRefs) -- Save original Data Refs and current dialogue position
return nil
end
if #strAns < 4 then -- If empty Data Refs then restore defaults
strRefs = loadRefs()
saveRefs(strRefs,fileRefs) -- Save reset Data Refs and current dialogue position
strErrs = strHelp -- V1.2
theAns = false
else
strRefs = strAns
saveRefs(strRefs,fileRefs) -- Save Data Refs and current dialogue position -- V1.2
theAns, strErrs = checkRefs(strRefs) -- Check Data Refs format
end
until theAns == true
saveRefs(strRefs,fileRefs) -- Save latest Data Refs and current dialogue position
return strRefs
end -- function getRefs
-- Make the Result Set arrays
local arrNum = {} -- Numerical order
local arrRec = {} -- Parent Record of Deleted Item
local arrBud = {} -- Parent Record buddy pointer
local arrRid = {} -- Parent Record Id
local arrDel = {} -- Text of Deleted Item
local arrPar = {} -- Parent Item of Deleted Item
local arrRef = {} -- Data Reference of Deleted Item
function makeResultSet(ptrDel,strRef) -- Compile the Result Set for Deleted Item matching Data Ref -- V1.2
local intCount = tblDataCounts[strRef]
if intCount <= 9 then -- Only list first 9 deletions per Data Ref
local ptrRec = fhNewItemPtr()
ptrRec:MoveToRecordItem(ptrDel) -- Get the Parent Record of Deleted Item
local strDel = ""
if ptrRec:IsSame(ptrDel) then strDel = " (deleted)" end -- Identify deleted record
table.insert( arrNum,(arrNum[#arrNum] or 0) + 1 ) -- Numerical order of entries
table.insert( arrRec,fhGetDisplayText(ptrRec,"","min")..strDel )-- Parent Record text
table.insert( arrBud,ptrRec:Clone() ) -- Parent Record buddy pointer
table.insert( arrRid,fhGetRecordId(ptrRec) ) -- Parent Record Id
table.insert( arrDel,fhGetDisplayText(ptrDel,"","min") ) -- Deleted Item text
ptrDel:MoveToParentItem(ptrDel)
table.insert( arrPar,ptrDel:Clone() ) -- Parent Item pointer
table.insert( arrRef,"%"..strRef.."%" ) -- Data Reference text
arrRef[strRef] = #arrRef
else
intCount = intCount - 9 -- Identify how many more like this
arrRef[arrRef[strRef]] = "%"..strRef.."% and "..intCount.." more like this..."
end
end -- function makeResultSet
-- Build the Data Tag and its Index
function buildIndex(dicRef,ptrTag) -- Cater for multiple tag instance index -- V1.2
local strTag = fhGetTag(ptrTag)
local intTag = ( dicRef[strTag] or 0 ) + 1 -- Count each instance of same tag
dicRef[strTag] = intTag
return strTag, "["..intTag.."]" -- Return tag name and its index
end -- function buildIndex
-- Process Fact or Tag
function processTag(ptrTag,dicTags)
if dicTags then -- Ignore if tag not listed
if #dicTags == 1 then
table.insert(arrDeleteList,ptrTag:Clone()) -- Delete final leaf tag
local strRef = dicTags[1]
tblDataCounts[strRef] = tblDataCounts[strRef] + 1 -- Count deletions
makeResultSet(ptrTag,strRef) -- V1.2
else
local dicRef = {} -- List counting multiple tag instances -- V1.2
ptrTag:MoveToFirstChildItem(ptrTag)
while ptrTag:IsNotNull() do
local strTag, strInd = buildIndex(dicRef,ptrTag) -- Cater for multiple tag instance index -- V1.2
processTag(ptrTag:Clone(),dicTags[strTag]) -- Search listed child tags without index -- V1.2
processTag(ptrTag:Clone(),dicTags[strTag..strInd]) -- Search listed child tags with an index -- V1.2
ptrTag:MoveNext()
end
end
end
end -- function processTag
-- Process File
function Main()
local strRefs = getRefs()
if not strRefs then return end
local dicType = {} -- Dictionary of nested data ref tags
for _, strRef in ipairs (split(strRefs,"\n")) do
strRef = strRef:match("^%%([^%%]+)%%")
if strRef then -- Found %data ref%
local dicType = dicType
local arrTag = split(strRef,".")
for intTag, strTag in ipairs (arrTag) do -- Convert data ref into nested dictionary table
if not dicType[strTag] then
if intTag == 1 then table.insert(dicType,strTag) end -- Note the record tag order -- V1.2
dicType[strTag] = {} -- Create next nested table
end
dicType = dicType[strTag]
end
table.insert(dicType,strRef) -- Save final leaf tag data ref
table.insert(tblDataCounts,strRef)
tblDataCounts[strRef] = 0 -- Zero data ref deletion count
end
end
local intRec = 0 -- Is progbar needed? -- V1.4
for _, strType in ipairs (dicType) do
local strRec = strType:match("^([_%u]+)") -- Extract record tag name
local ptrRec = fhNewItemPtr()
ptrRec:MoveToFirstRecord(strRec)
while ptrRec:IsNotNull() do -- Count each record type -- V1.4
intRec = intRec + 1
ptrRec:MoveNext()
end
end
if intRec > 10000 then progbar.Start("Finding Unwanted Fields",intRec) end -- V1.4
intRec = 0
for _, strType in ipairs (dicType) do -- Search all matching data refs in record tag order -- V1.2
local dicTags = dicType[strType]
local strRec = strType:match("^([_%u]+)") -- Extract record tag name -- V1.2
local intRid = tonumber(strType:match("%[(%d+)%]$")) -- Extract record id if any -- V1.2
local ptrRec = fhNewItemPtr()
ptrRec:MoveToFirstRecord(strRec)
while ptrRec:IsNotNull() do -- Search each record type
progbar.Step(1) -- V1.4
if progbar.Stop() then break end
if not intRid or intRid == fhGetRecordId(ptrRec) then -- with matching Record Id -- V1.2
if #dicTags == 1 then
table.insert(arrDeleteList,ptrRec:Clone()) -- Delete record and count
tblDataCounts[strType] = tblDataCounts[strType] + 1
makeResultSet(ptrRec,strType) -- V1.2
else
local ptrTag = fhNewItemPtr()
local dicRef = {} -- List counting multiple tag instances -- V1.2
ptrTag:MoveToFirstChildItem(ptrRec)
while ptrTag:IsNotNull() do
local strTag, strInd = buildIndex(dicRef,ptrTag) -- Cater for multiple tag instance index -- V1.2
processTag(ptrTag:Clone(),dicTags[strTag]) -- Search specific tags without index -- V1.2
processTag(ptrTag:Clone(),dicTags[strTag..strInd])-- Search specific tags with an index -- V1.2
if fhIsFact(ptrTag) then
processTag(ptrTag:Clone(),dicTags["Fact"]) -- Search generic Facts without index -- V1.2
processTag(ptrTag:Clone(),dicTags["Fact"..strInd])-- Search generic Facts with an index -- V1.2
end
ptrTag:MoveNext()
end
end
if intRid then break end -- Escape if single Record Id defined -- V1.2
end
ptrRec:MoveNext()
end
end
progbar.Close() -- V1.4
local strZero = ""
for _, strRef in ipairs (tblDataCounts) do -- Find unused/invalid data refs
if tblDataCounts[strRef] == 0 then
strZero = strZero.."%"..strRef.."%\n"
end
end
if #strZero > 0 then
strZero = "\n\nThese data refs were not used:\n"..strZero
end
local strAns = fhMessageBox("Please Confirm Changes:\n\n"..#arrDeleteList.." items will be deleted.\n\nTo reverse those deletions use:\nEdit > Undo Plugin Updates"..strZero,"MB_OKCANCEL")
if strAns == "OK" then
for _, ptrItem in ipairs(arrDeleteList) do
fhDeleteItem(ptrItem)
end
if #arrRec > 0 then -- Output the Result Set details -- V1.2
fhOutputResultSetTitles(strPluginName)
fhOutputResultSetColumn("No." ,"integer",arrNum,#arrRec, 20)
fhOutputResultSetColumn("Parent Record" ,"text",arrRec,#arrRec,200)
fhOutputResultSetColumn("Parent Record" ,"item",arrBud,#arrRec,200,"align_left",0,true,"default","buddy")
fhOutputResultSetColumn("RecId" ,"integer",arrRid,#arrRec, 25)
fhOutputResultSetColumn("Data Reference","text",arrRef,#arrRec,200)
fhOutputResultSetColumn("Parent Item" ,"item",arrPar,#arrRec,200)
fhOutputResultSetColumn("Deleted Item" ,"text",arrDel,#arrRec,400)
end
end
end -- function Main
fhInitialise(5,0,8,"save_recommended") -- V5.0.8 for sticky settings with scope
Main()
Source:Clean-Unwanted-Fields-2.fh_lua