Plugin Store

Lumped Source Splitter.fh_lua

--[[
@Title:       Lumped Source Splitter
@Type:        Standard
@Author:      Mark Draper
@Version:     1.5     
@LastUpdated: 25 Apr 2025
@Licence:     This plugin is copyright (c) 2025 Mark Draper and is licensed under the MIT License which
 is hereby incorporated by reference (see https://pluginstore.family-historian.co.uk/fh-plugin-licence)
@Description: Converts a lumped source to individual split sources. Citations are regarded as equivalent
if they have identical Where within Source and Text from Source (or Citation level fields for Templated
Sources), and the same attached Media or Note Records. All of these items are moved to the new split
source and the original lumped citation deleted.
]]

--[[

Version 1.2 (Oct 2021)
	- Initial Plugin Store version, equivalent to FHUG version 1.1.3
Version 1.3 (Jun 2023)
	- Redesigned treatment of templated sources, moving citation data to source level
	- Improved messaging and reporting
	- Numerous improvements in coding to speed up execution time
Version 1.4 (Jul 2023) 
	- Citation notes now ignored for splitting
	- Minor menu presentation improvements
Version 1.4.1 (Feb 2024)
	- Corrects bug whereby media are retained in citation
Version 1.5 (Apr 2025)
	- Support for multiple monitors added
	- Support for enhanced tooltips added
]]

fhInitialise(7, 0, 15, 'save_recommended')
require('iuplua')
require('iupluaimglib')
fhu = require('fhUtils')
fhu.setIupDefaults()
iup.SetGlobal('IMAGESTOCKSIZE', 32)

-- *********************************************************************

function main()

	gblReport = {}				-- initialize table for plugin results

	local selection
	local msgTitle = 'This plugin converts a lumped source into individual split sources and recombines\n' ..
		'duplicated citations created during import from another genealogy application'
	local lblTitle = iup.label{title = msgTitle}
	local btnSingle = iup.button{title = 'Single Source', padding = '10x3',
			tip = 'Select a single source for splitting'}
	local btnMultiple = iup.button{title = 'Multiple Sources',
			tip = 'Select multiple sources for splitting'}
	local btnTemplate = iup.button{title = 'Split by Template',
			tip = 'Split all sources with same template'}
	local btnHelp = iup.button{title = 'Help',
			action = function(self) fhShellExecute('https://pluginstore.family-historian.co.uk/page/help/lumped-source-splitter') end,
			tip = 'Show plugin help page'}
	local btnClose = iup.button{title = 'Close',
			action = function(self) return iup.CLOSE end, padding = '10x3',
			tip = 'Close plugin'}

	-- define enhanced tool tips

	local enhanced = true					-- comment out this line if enhanced tool tips are not required
	if enhanced then
		local tblB = {btnSingle, btnMultiple, btnTemplate, btnHelp, btnClose}
		for _, control in ipairs(tblB) do
			control.TipBalloon = 'YES'
			control.TipBalloonTitleIcon = 1			-- modify individually if different
		end
		btnSingle.TipBalloonTitle = 'Single Source'
		btnMultiple.TipBalloonTitle = 'Multiple Sources'
		btnTemplate.TipBalloonTitle = 'Split by Template'
		btnHelp.TipBalloonTitle = 'Help'
		btnClose.TipBalloonTitle = 'Close'
	end

	local SplitButtons = iup.vbox{iup.fill{}, btnSingle, btnMultiple, btnTemplate, iup.fill{};
			margin = '0x10', normalizesize = 'Both', padding = 10, gap = 20}

	local AdminButtons = iup.hbox{iup.fill{}, btnHelp, btnClose, iup.fill{};
			margin = '0x10', normalizesize = 'Both', padding = 10, gap = 50}

	local vbox = iup.vbox{lblTitle, SplitButtons, AdminButtons; alignment = 'ACENTER'; margin = '20x20'}

	local dialog = iup.dialog{vbox; title = 'Lumped Source Splitter (1.4.1)', 
			resize = 'No', minbox = 'No'}
	if fhGetAppVersion() > 6 then
		iup.SetAttribute(dialog, 'NATIVEPARENT', fhGetContextInfo('CI_PARENT_HWND'))
	end

	function btnSingle:action()
		local tblS = fhPromptUserForRecordSel('SOUR', 1, dialog.HWND)
		ProcessSources(tblS)
		return iup.CLOSE
	end

	function btnMultiple:action()
		local tblS = fhPromptUserForRecordSel('SOUR', -1, dialog.HWND)
		ProcessSources(tblS)
		return iup.CLOSE
	end

	function btnTemplate:action()
		local tblT = fhPromptUserForRecordSel('_SRCT', 1, dialog.HWND)
		SelectSources(tblT)
		return iup.CLOSE
	end

	dialog:popup()

	-- report plugin actions

	if #gblReport == 0 then return end

	local tblS = {}
	local tblT = {}
	local tblA = {}
	local tblC = {}
	local tblR = {}

	local p = fhNewItemPtr()
	for _, v in pairs(gblReport) do
		p:MoveToRecordById('SOUR', v.ID)
		table.insert(tblS, p:Clone())
		table.insert(tblT, fhGetItemPtr(p, '~._SRCT'))
		table.insert(tblA, v.All)
		table.insert(tblC, v.Citations)
		table.insert(tblR, fhCallBuiltInFunction('LinksTo', p))
	end

	fhOutputResultSetTitles('Split Sources')
	fhOutputResultSetColumn('Source', 'item', tblS, #tblS, 120, 'align_left', 2)
	fhOutputResultSetColumn('Template', 'item', tblT, #tblT, 120)
	fhOutputResultSetColumn('All Citations', 'integer', tblA, #tblA, 60, 'align_mid')
	fhOutputResultSetColumn('Lumped Citations', 'integer', tblC, #tblC, 80, 'align_mid', 1, false)
	fhOutputResultSetColumn('Remaining Citations', 'integer', tblR, #tblR, 80, 'align_mid')
end

-- ************************************************************************** --

function SelectSources(tblT)

	if #tblT == 0 then return end				-- nothing selected

	local tblS = {}
	local p = fhNewItemPtr()
	p:MoveToFirstRecord('SOUR')
	while p:IsNotNull() do
		local pL = fhGetItemPtr(p, '~._SRCT')
		local pT = fhGetValueAsLink(pL)
		if pT:IsSame(tblT[1]) then
			table.insert(tblS, p:Clone())
		end
		p:MoveNext()
	end
	ProcessSources(tblS)
 end

-- ************************************************************************** --

function ProcessSources(tblS)

	local tblX = {}
	local tblTitleFields = {}

	if #tblS == 0 then return end						-- nothing selected

	local tblAllCitations = PopulateCitations()			-- generate master citation list
	local changed

	-- now start processing selected sources

	for _, pSource in ipairs(tblS) do

		-- find citations to selected source

		local tblCitations = GetCitations(pSource, tblAllCitations)
		local CitationCount = 0
		for k, _ in pairs(tblCitations) do CitationCount = CitationCount + 1 end

		-- report results

		local tblT = {}
		tblT.ID = fhGetRecordId(pSource)
		tblT.All = fhCallBuiltInFunction('LinksTo', pSource)
		tblT.Citations = CitationCount
		table.insert(gblReport, tblT)

		-- create new sources

		if CitationCount > 0 then
			changed = true
			local pL = fhGetItemPtr(pSource, '~._SRCT')		-- check for templated source
			local pT = fhNewItemPtr()
			if pL:IsNotNull() then
				pT = ProcessTemplate(fhGetValueAsLink(pL))
			end
			for _, Citations in pairs(tblCitations) do
				CreateNewSource(pSource, Citations, pT)
			end
			fhUpdateDisplay()

			-- warn if source still has citations

			if fhCallBuiltInFunction('LinksTo', pSource) > 0 then
				local msg = fhGetDisplayText(pSource) .. ' still has remaining citations. This is ' ..
						'usually caused by citations that do not contain any detailed content.'
				MessageBox(msg, 'OK', 'WARNING')
			end
		end
	end

	-- warn if no changes

	if not changed then
		MessageBox('No lumped citations in selection.', 'OK', 'INFORMATION')
	else
		MessageBox('Processing completed successfully.', 'OK', 'INFORMATION')
	end
end

-- ************************************************************************** --

function PopulateCitations(ItemCount, tblAllCitations)

	-- creates a master table of all source citations organised by source,
	-- but not yet any analysis of those citations

	local ItemCount = 0
	local tblC = {}
	local p = fhNewItemPtr()

	p:MoveToFirstRecord('SOUR')
	while p:IsNotNull() do
		tblC[fhGetRecordId(p)] = {}
		ItemCount = ItemCount + fhCallBuiltInFunction('LinksTo', p)
		p:MoveNext()
	end

	-- populate source table

	local Count = 0
	local Now
	local LastPing = os.clock()

	-- display progress bar

	local ProgressBar = iup.progressbar{expand = 'HORIZONTAL', max = ItemCount}
	local Label = iup.label{expand = 'HORIZONTAL', alignment = 'ACENTER'}
	local imgIcon = iup.animatedlabel{animation = 'IUP_CircleProgressAnimation'}
	iup.SetAttribute(imgIcon, 'START', 'YES')

	local vbox = iup.vbox{ProgressBar, imgIcon, Label; gap = 10, alignment = 'ACENTER', margin = '0x15'}
	local dialog = iup.dialog{vbox; title = 'Counting citations...', dialogframe = 'YES', border = 'YES',
			size = '300x', menubox = 'NO'}
	dialog:showxy(iup.CENTER, iup.CENTER)	-- Put up Progress Display

	for _, RecType in ipairs({'INDI', 'FAM', 'OBJ', 'NOTE'}) do
		p:MoveToFirstRecord(RecType)
		while p:IsNotNull() do
			if fhGetTag(fhGetValueAsLink(p)) == 'SOUR' then
				table.insert(tblC[fhGetRecordId(fhGetValueAsLink(p))], p:Clone())
				Count = Count + 1
			end
			collectgarbage('step', 0)
			if Count%200 == 0 then
				Label.title = 'Records checked: ' .. string.format('%.1f', Count/ItemCount*100) .. '%'
				ProgressBar.value = Count
			end
			Now = os.clock()
			if Now - LastPing > 2 then
				fhExhibitResponsiveness()
				LastPing = Now
			end
			p:MoveNextSpecial()
		end
	end
	dialog:destroy()
	return tblC
end

-- ************************************************************************** --

function ProcessTemplate(pT)

	local match
	local p = fhNewItemPtr()

	local FieldProfile = GetFields(pT, false)

	-- compare template with existing ones

	p:MoveToFirstRecord('_SRCT')
	while p:IsNotNull() do
		if GetFields(p, true) == FieldProfile then					-- templates match
			return(p:Clone())
		end
		p:MoveNext()
	end

	-- no match obtained, so copy template and convert citation fields to source

	local pNew = fhCreateItem('_SRCT')
	CopyChildren(pT, pNew)					-- Copy all details from template to new record

	-- modify new copy

	local TemplateName = fhGetItemText(pT, '~.NAME')
	fhSetValueAsText(fhGetItemPtr(pNew, '~.NAME'), TemplateName .. ' (Split copy)')

	-- create a basic title format if none exists already

	local pTitle = fhGetItemPtr(pNew, '~.TITL')
	local RecordTitle = ''
	if pTitle:IsNotNull() then
		RecordTitle = fhGetValueAsText(pTitle)
	else															-- automatic title not defined
		pTitle = fhCreateItem('TITL', pNew)
		RecordTitle = fhGetItemText(pT, '~.NAME')
		p:MoveTo(pNew, '~.FDEF')
		while p:IsNotNull() do
			local pCit = fhGetItemPtr(p, '~.CITN')
			if pCit:IsNull() then									-- existing source level field
				RecordTitle = RecordTitle .. ' - {' .. fhGetItemText(p, '~.CODE') .. '}'
			end
			p:MoveNext('SAME_TAG')
		end
		fhSetValueAsText(pTitle, RecordTitle)
	end

	-- move citation fields to source level and add to title format

	local tblX = {}
	p:MoveTo(pNew, '~.FDEF')
	while p:IsNotNull() do
		local pCit = fhGetItemPtr(p, '~.CITN')
		if pCit:IsNotNull() then									-- add field to auto title
			RecordTitle = RecordTitle .. ' - {' .. fhGetItemText(p, '~.CODE') .. '}'
			table.insert(tblX, pCit:Clone())					-- delete as no longer a citation field
		end
		p:MoveNext('SAME_TAG')
	end

	fhSetValueAsText(pTitle, RecordTitle)
	for _, pX in ipairs(tblX) do fhDeleteItem(pX) end
	return pNew
end

-- ************************************************************************** --

function GetFields(pT, IncludeCitation)

	-- return string representing template field profile

	local p = fhNewItemPtr()
	local tblFields = {}
	p:MoveTo(pT, '~.FDEF')
	while p:IsNotNull() do
		local F = fhGetItemText(p, '~.NAME') .. fhGetItemText(p, '~.TYPE')
		if IncludeCitation and fhGetItemPtr(p, '~.CITN'):IsNotNull() then F = F .. 'Citation' end
		table.insert(tblFields, F)
		p:MoveNext('SAME_TAG')
	end
	table.sort(tblFields)
	return table.concat(tblFields)
end

-- ************************************************************************** --

function GetCitations(pSource, tblAllCitations)

	-- returns a table of citation fingerprints for all citations to the defined source

	local tblCitations = {}

	for _, Citation in ipairs(tblAllCitations[fhGetRecordId(pSource)]) do
		local k = GetCitationFingerprint(Citation)
		if k then
			if not tblCitations[k] then tblCitations[k] = {} end
			table.insert(tblCitations[k], Citation:Clone())
		end
	end
	return tblCitations
end

-- ************************************************************************** --

function GetCitationFingerprint(Citation)

	-- returns a text representation of the specified citation

	local p = fhNewItemPtr()
	local tblK = {}		-- text representation of pointers for easy comparison

	for _, Tag in ipairs({'~.PAGE', '~.DATA.TEXT', '~.OBJE', '~._FIELD'}) do
		p:MoveTo(Citation, Tag)
		while p:IsNotNull() do
			table.insert(tblK, fhGetDisplayText(p))
			p:MoveNext('SAME_TAG')
		end
	end
	if #tblK > 0 then return table.concat(tblK) end
end

-- ************************************************************************** --

function CreateNewSource(pSource, Citations, pT)

	local p = fhNewItemPtr()
	local CommonCitation = Citations[1]		-- lumped elements of citation are all the same, so use first one

	-- create new Source Record as copy of original lumped source

	local pNewSource = fhCreateItem('SOUR')
	CopyChildren(pSource, pNewSource)

	-- change template or modify generic source title

	if pT:IsNotNull() then
		fhSetValueAsLink(fhGetItemPtr(pNewSource, '~._SRCT'), pT)
	else
		local Modifier = fhGetItemText(CommonCitation, '~.PAGE')
		p:MoveTo(pNewSource, '~.TITL')
		if p:IsNotNull() and Modifier ~= '' then
			fhSetValueAsText(p, fhGetValueAsText(p) .. ': ' .. Modifier)
		end
		p:MoveTo(pNewSource, '~.ABBR')
		if p:IsNotNull() and Modifier ~= '' then
			fhSetValueAsText(p, fhGetValueAsText(p) .. ': ' .. Modifier)
		end
	end

	-- copy common citation data to source

	p:MoveToFirstChildItem(CommonCitation)
	while p:IsNotNull() do
		local Tag = fhGetTag(p)
		if Tag == 'PAGE' then
			local pPUBL = fhCreateItem('PUBL', pNewSource)
			fhSetValue_Copy(pPUBL, p)
		elseif Tag == 'DATA' then
			local pCitText = fhNewItemPtr()
			pCitText:MoveTo(p, '~.TEXT')
			if pCitText:IsNotNull() then
				local pTEXT = fhCreateItem('TEXT', pNewSource)
				fhSetValue_Copy(pTEXT, pCitText)
			end
		elseif Tag == 'OBJE' then
			local pNew = fhCreateItem(Tag, pNewSource)
			fhSetValue_Copy(pNew, p)
		elseif Tag == '_FIELD' then
			local pNew = fhCreateItem(fhGetMetafieldShortcut(p), pNewSource)
			fhSetValue_Copy(pNew, p)
		end
		p:MoveNext()
	end

	fhSrcEnableAutoTitle(pNewSource, true)			-- update title for templated source

	-- source is now fully prepared, so create individual citations to it

	local tblXcit = {}		-- deletion bin for redundant citations

	for _, Citation in pairs(Citations) do
		local pParent = fhNewItemPtr()
		pParent:MoveToParentItem(Citation)

		-- create new source link

		local pNew = fhCreateItem('SOUR', pParent)
		fhSetValueAsLink(pNew, pNewSource)

		-- delete redundant fields in lumped source citation

		local tblX = {}				-- deletion bin for redundant fields
		local tblFields = {'~.PAGE', '~.DATA.TEXT', '~.OBJE', '_FIELD'}
		for _, Field in ipairs(tblFields) do
			p:MoveTo(Citation, Field)
			while p:IsNotNull() do
				table.insert(tblX, p:Clone())
				p:MoveNext('SAME_TAG')
			end
		end

		-- delete DATA field if no children

		p:MoveTo(Citation, '~.DATA')
		if not fhHasChildItem(p) then table.insert(tblX, p:Clone()) end

		-- delete field pointers now as next step is dependent on this

		for _, pX in ipairs(tblX) do fhDeleteItem(pX) end

		-- copy any remaining lumped citation fields to new citation

		p:MoveToFirstChildItem(Citation)
		while p:IsNotNull() do
			local Tag = fhGetTag(p)
			local pNewField = fhCreateItem(Tag, pNew)
			fhSetValue_Copy(pNewField, p)
			p:MoveNext()
		end

		-- process any citation Entry Date

		p:MoveTo(Citation, '~.DATA.DATE')
		if p:IsNotNull() then
			local pDATA = fhNewItemPtr()
			pDATA:MoveTo(pNew, '~.DATA')
			local pDATE = fhCreateItem('DATE', pDATA)
			fhSetValue_Copy(pDATE, p)
		end

		-- finally, delete old lumped citation

		table.insert(tblXcit, Citation:Clone())
	end

	for _, pX in ipairs(tblXcit) do fhDeleteItem(pX) end		-- empty the citation bin
end

-- ************************************************************************** --

function MessageBox(Message, Buttons, Icon, Title, Default)

	-- replaces built-in function with custom version containing more options

	local msgdlg = iup.messagedlg{value = Message, buttons = Buttons, dialogtype = Icon, 
			title = Title or 'Lumped Source Splitter (1.4.1)', buttondefault = Default}

	-- display message box and return selection

	msgdlg:popup()

	return tonumber(msgdlg.ButtonResponse)
end

-- *********************************************************************

function CopyChildren(ptrSource,ptrTarget)

	local ptrFrom = fhNewItemPtr()
	ptrFrom = ptrSource:Clone()
	ptrFrom:MoveToFirstChildItem(ptrFrom)
	while ptrFrom:IsNotNull() do
		CopyBranch(ptrFrom,ptrTarget)
		ptrFrom:MoveNext()
	end
end

-- *********************************************************************

function CopyBranch(ptrSource, ptrTarget)

	local strTag = fhGetTag(ptrSource)
	if strTag == '_FMT' then return end		-- Skip rich text format code
	if strTag == '_FIELD' then				-- Substitute metafield shortcut
		strTag = fhGetMetafieldShortcut(ptrSource)
	end

	local ptrNew = fhCreateItem(strTag, ptrTarget,true)
	if ptrNew:IsNull() then return end		-- Escape if item not created
	fhSetValue_Copy(ptrNew, ptrSource)
	CopyChildren(ptrSource, ptrNew)
end

-- *********************************************************************

main()

Source:Lumped-Source-Splitter-6.fh_lua