From 449a297d8bb8b2a52b4ec1623140a4ab6131beb1 Mon Sep 17 00:00:00 2001 From: lksz Date: Fri, 29 Jan 2021 18:26:39 -0500 Subject: [PATCH] Main ConvertTo-Zip with a few additions and fixes + ConvertTo-Zip, originally created to mass convert CBRs to CBZs It will only be avilable when 7-zip is available (7z executable) Features: Shows progress, can work recursively (preserving folder structure), knows to move completed files. + Show-Progress: Shorthand for common progress related output manipulation (calculation of completion based on time, or item count, and verbose output if needeed) + ConvertFrom-TimeSpan: string output from timespan, used by Show-Progress + Get-Path updated to be more efficient (no need for exception handling) + PathProcessingFunctions loads a utility function GetShellSafePath * Repair-Permissions added some status reporting --- 7zip/ConvertTo-Zip.ps1 | 125 ++++++++++++++++++ 7zip/_.package.json | 14 ++ base.linux/Repair-Permissions.ps1 | 45 +++++-- base/ConvertFrom-TimeSpan.ps1 | 15 +++ base/Get-Path.ps1 | 31 ++++- base/Show-Progress.ps1 | 61 +++++++++ .../profile.d/PathProcessingFunctions.ps1.ps1 | 4 + 7 files changed, 276 insertions(+), 19 deletions(-) create mode 100644 7zip/ConvertTo-Zip.ps1 create mode 100644 7zip/_.package.json create mode 100644 base/ConvertFrom-TimeSpan.ps1 create mode 100644 base/Show-Progress.ps1 create mode 100644 base/profile.d/PathProcessingFunctions.ps1.ps1 diff --git a/7zip/ConvertTo-Zip.ps1 b/7zip/ConvertTo-Zip.ps1 new file mode 100644 index 0000000..6907438 --- /dev/null +++ b/7zip/ConvertTo-Zip.ps1 @@ -0,0 +1,125 @@ +[CmdletBinding(SupportsShouldProcess)]param( + [string]$SourcePath=".", + [string]$SourceSet="*.rar", + [switch]$Recurse, + [string]$ZipExtension=".zip", + [string]$ZipDestPath="./zipped", + [string]$ZipOptions="-mm=Deflate -mfb=258 -mpass=15", + [string]$MoveProcessedTo="done", + [switch]$Flatten, + [switch]$Force +) + +function Get7zExtractCmd { + param([string]$ArchivePath,[string]$TmpPath) + $local:7zCmd = "7z " + if( $Flatten ) { + $7zCmd += "e " + } else { + $7zCmd += "x " + } + $7zCmd += "-o$(Get-ShellSafePath $TmpPath) $(Get-ShellSafePath $ArchivePath)" + + $7zCmd +} + +function Get7zCompressCmd { + param([string]$ArchivePath),[string]$CompressOptions + $local:7zCmd = "7z a -sdel " + if( -not $Flatten ) { $7zCmd += "-r " } + $7zCmd += "$CompressOptions $(Get-ShellSafePath $ArchivePath)" + + $7zCmd +} + +# Create a new temporary dir +$local:DestPathRoot = Get-Path "$ZipDestPath/" +$local:TempPath = Join-Path $DestPathRoot ([System.IO.Path]::GetRandomFileName()) +$local:DonePath = [string]::Empty +if( $MoveProcessedTo ) { + $DonePath = Get-Path $MoveProcessedTo +} +Write-Verbose "`n`tTempPath: $TempPath;`n`tDestPathRoot: $DestPathRoot;`n`tDonePath: $DonePath" + +$local:runTimeSpan = [System.Diagnostics.Stopwatch]::StartNew() +$local:lastStatus = $runTimeSpan.Elapsed + +$SourcePath = Get-Path "$SourcePath/" +$local:srcSet = Get-ChildItem -Recurse:$Recurse -LiteralPath $SourcePath -Include $SourceSet + +$local:progress = [ordered]@{ + Activity = "ConvertTo-Zip $SourcePath" + ItemCount = $srcSet.Count + ItemProgress = 0 + Elapsed = $runTimeSpan.Elapsed +} + +foreach( $local:srcFile in $srcSet ) { + Write-Verbose "Processing $($srcFile.Name)..." + $progress.ItemProgress++ + $progress.Elapsed = $runTimeSpan.Elapsed + if( ($runTimeSpan.Elapsed - $lastStatus).TotalSeconds -gt 3 ) { + Show-Progress @progress -Status "Processing $($srcFile.Name)..." + } + if( Test-Path -LiteralPath $TempPath ) { + Remove-Item -Recurse -Force -LiteralPath $TempPath -ErrorAction Stop + } + + $local:relPath = $srcFile.DirectoryName.Replace($SourcePath,[string]::Empty) + $local:destPath = $srcFile.DirectoryName.Replace($SourcePath,$DestPathRoot) + $local:destZipPath = $(Join-Path $destPath "$($srcFile.BaseName)$ZipExtension") + + $local:ActionStatus = "Failed" + Write-Verbose "Dest: $destZipPath" + if( -not $Force -and (Test-Path -LiteralPath $destZipPath) ) { + Write-Verbose "Skipping, dest already exists: `"$($destZipPath.Replace($DestPathRoot,[string]::Empty))`"" + $ActionStatus = "Skipped" + } else { + $local:nextCmd = Get7zExtractCmd -ArchivePath $srcFile.FullName -TmpPath $TempPath + $null = Invoke-ExpressionEx $nextCmd + if( -not (Test-Path -PathType Container -LiteralPath $TempPath) ) { + Write-Warning "Failed to convert file:`n`t$($srcFile.FullName)" + $ActionStatus = "Failed" + } else { + Push-Location -LiteralPath $TempPath -ErrorAction Stop + + $local:destFile = Join-Path $destPath "$([System.io.path]::GetRandomFileName()).zip" + $nextCmd = Get7zCompressCmd -ArchivePath $destFile + $null = Invoke-ExpressionEx $nextCmd + Pop-Location + + if( -not (Test-Path -LiteralPath $destPath ) ) { + New-Item -Type Directory -LiteralPath $destPath + } + $null = Move-Item -Force:$Force -LiteralPath $destFile $destZipPath + + $ActionStatus = "Processed" + } + } + + if( Test-Path -PathType Container -LiteralPath $TempPath ) { + Remove-Item -Recurse -Force -LiteralPath $TempPath + } + + if( $DonePath ) { + $local:doneFinalPath = Join-Path $DonePath $relPath + if( -not (Test-Path -LiteralPath $doneFinalPath) ) { $null = New-Item -Type Directory $doneFinalPath } + $local:moveParams = [ordered]@{ + LiteralPath=$srcFile.FullName + Destination=$doneFinalPath + } + if( Test-Path (Join-Path $doneFinalPath $($srcFile.Name))) { + $moveParams.Confirm = $true + $moveParams.Force = $true + } + Move-Item @moveParams + } + + [PSCustomObject]([ordered]@{ + Action = $ActionStatus + SrcMoved = $(if($DonePath){ $true } else {$false}) + Path = $relPath + File = $srcFile.BaseName + }) +} + diff --git a/7zip/_.package.json b/7zip/_.package.json new file mode 100644 index 0000000..e5a62e8 --- /dev/null +++ b/7zip/_.package.json @@ -0,0 +1,14 @@ +{ + "package": { + "Condition": [ + { + "custom": "Get-Command 7z | Where-Object CommandType -eq 'Application'", + "System": null, + "Hostname": null, + "Username": null, + "Logic": 0 + } + ], + "Name": "7zip" + } +} diff --git a/base.linux/Repair-Permissions.ps1 b/base.linux/Repair-Permissions.ps1 index 3cf76db..87823a1 100644 --- a/base.linux/Repair-Permissions.ps1 +++ b/base.linux/Repair-Permissions.ps1 @@ -19,9 +19,16 @@ begin { $User = $env:SZ_DOCKER_PUID ?? $env:USER; } $User = "$(& id --user --name $User)"; - + if( [string]::IsNullOrWhiteSpace($User) ) { + $User = $env:USER; + } + $User = "$(& id --user --name $User)"; + if( [string]::IsNullOrWhiteSpace($Group) ) { $Group = $env:SZ_DOCKER_GUID ?? "$(& id --group --name $User)" + } + if( [string]::IsNullOrWhiteSpace($Group) ) { + $Group = "$(& id --group --name $User)" } if( [string]::IsNullOrWhiteSpace($FilePermission) ) { @@ -36,23 +43,37 @@ begin { } process { + $local:Dirs = @() + $local:Files = @() + $local:item = $Path | Get-Item + $Path | Get-Item | ForEach-Object { $local:p = $_.FullName $local:nextCmd = "" - if( $_.Attributes -band [System.IO.FileAttributes]::Directory ) { - $nextCmd = "Get-ChildItem $p $cmdArgs | ForEach-Object { & chown ${User}:${Group} `$_ }" - if( $PSCmdlet.ShouldProcess( "Set $p ownership to ${User}:${Group}`n$nextCmd`n" ) ) { - Invoke-ExpressionEx -sudo:$sudo $nextCmd - } - } - - $nextCmd = "Get-ChildItem $p -Directory $cmdArgs | ForEach-Object { & chmod $DirPermission `$_ }" - if( $PSCmdlet.ShouldProcess( "Set $p dirs permissions to $DirPermission`n$nextCmd`n" ) ) { + + $nextCmd = "(@(Get-Item '$p') + (Get-ChildItem '$p' $cmdArgs)) | " + + "ForEach-Object { & chown ${User}:${Group} `$_.FullName }" + $local:msg = "Set $p ownership to ${User}:${Group}" + if( $PSCmdlet.ShouldProcess( "$msg`n$nextCmd`n" ) ) { + Write-Host -ForegroundColor Cyan $msg Invoke-ExpressionEx -sudo:$sudo $nextCmd } - $nextCmd = "Get-ChildItem $p -File $cmdArgs | ForEach-Object { & chmod $FilePermission `$_ }" - if( $PSCmdlet.ShouldProcess( "Set $p file permissions to $FilePermission`n$nextCmd`n" ) ) { + $nextCmd = "(Get-ChildItem '$p' -Directory $cmdArgs)) | ForEach-Object { & chmod $DirPermission `$_ }" + if( $_.Attributes -band 'Directory' ) { $nextCmd = "@(Get-Item '$p') + $nextCmd" } + $nextCmd = "($nextCmd" + $msg = "Set $p dirs permissions to $DirPermission" + if( $PSCmdlet.ShouldProcess( "$msg`n$nextCmd`n" ) ) { + Write-Host -ForegroundColor Cyan $msg + Invoke-ExpressionEx -sudo:$sudo $nextCmd + } + + $nextCmd = "(Get-ChildItem '$p' -File $cmdArgs)) | ForEach-Object { & chmod $FilePermission `$_ }" + if( -not ($_.Attributes -band 'Directory') ) { $nextCmd = "@(Get-Item '$p') + $nextCmd" } + $nextCmd = "($nextCmd" + $msg = "Set $p file permissions to $FilePermission" + if( $PSCmdlet.ShouldProcess( "$msg`n$nextCmd`n" ) ) { + Write-Host -ForegroundColor Cyan $msg Invoke-ExpressionEx -sudo:$sudo $nextCmd } } diff --git a/base/ConvertFrom-TimeSpan.ps1 b/base/ConvertFrom-TimeSpan.ps1 new file mode 100644 index 0000000..1c10416 --- /dev/null +++ b/base/ConvertFrom-TimeSpan.ps1 @@ -0,0 +1,15 @@ +param([TimeSpan]$TimeSpan, [switch]$NoSeconds) +$local:out = [string]::Empty +if( $TimeSpan.Days ) { + $out = "$($TimeSpan.Days)d " +} +if( $out -or $TimeSpan.Hours ) { + $out += "$($TimeSpan.Hours)h " +} +if( $out -or $TimeSpan.Minutes -or ($NoSeconds -and -not $out) ) { + $out += "$($TimeSpan.Minutes)m " +} +if( -not $NoSeconds -and ($TimeSpan.Seconds -or -not $out) ) { + $out += "$($TimeSpan.Seconds)s" +} +$out.Trim(); diff --git a/base/Get-Path.ps1 b/base/Get-Path.ps1 index 49c9a92..37d6cfe 100644 --- a/base/Get-Path.ps1 +++ b/base/Get-Path.ps1 @@ -1,11 +1,28 @@ [CmdletBinding()]param([string]$Path) -try { - if( $Path[0] -eq '@' ) { - $Path = Join-Path $MyPSScriptRoot $Path.Substring(1) +function Get-FullPath { + param([string]$Path) + + if([System.IO.Path]::IsPathRooted($Path)){ + [System.IO.Path]::GetFullPath($Path) + }else{ + [System.IO.Path]::GetFullPath((Join-Path $PWD $Path)) } - get-item $Path -Force -ErrorAction Stop | - Select-Object -ExpandProperty FullName -} catch { - $_.targetObject } + +if( $Path[0] -eq '@' ) { + $Path = Join-Path $MyPSScriptRoot $Path.Substring(1) +} + +if([System.IO.Path]::IsPathRooted($Path)){ + [System.IO.Path]::GetFullPath($Path) +}else{ + [System.IO.Path]::GetFullPath((Join-Path $PWD $Path)) +} + +# try { +# get-item $Path -Force -ErrorAction Stop | +# Select-Object -ExpandProperty FullName +# } catch { +# $_.targetObject +# } diff --git a/base/Show-Progress.ps1 b/base/Show-Progress.ps1 new file mode 100644 index 0000000..076f67a --- /dev/null +++ b/base/Show-Progress.ps1 @@ -0,0 +1,61 @@ +[CmdletBinding()]param( + [Parameter(Mandatory, Position=0)] + [string]$Activity, + [string]$Status, + [Parameter(ParameterSetName="Calculate",Mandatory)] + [int]$ItemProgress, + [Parameter(ParameterSetName="Calculate")] + [int]$ItemCount, + [Parameter(ParameterSetName="PercentComplete",Mandatory)] + [int]$PercentComplete, + [Parameter(ParameterSetName="SecondsRemaining",Mandatory)] + [int]$SecondsRemaining, + [Parameter(ParameterSetName="Elapsed",Mandatory)] + [Parameter(ParameterSetName="Calculate")] + [Timespan]$Elapsed, + [Parameter(ParameterSetName="Elapsed")] + [int]$TotalSeconds, + [string]$CurrentOperation, + [Parameter(ParameterSetName="Completed",Mandatory)] + [switch]$Completed +) + +$local:progressParams = [ordered]@{ + Activity = $Activity +} +if( $Status ) { $progressParams.Status = $Status } +if( $PercentComplete ) { $progressParams.PercentComplete = $PercentComplete } +if( $SecondsRemaining ) { $progressParams.SecondsRemaining = $SecondsRemaining } +if( $ItemCount ) { $progressParams.PercentComplete = (100.0 * $ItemProgress / $ItemCount) } +if( $CurrentOperation ) { $progressParams.CurrentOperation = $CurrentOperation } +if( $Completed ) { $progressParams.Completed = $true } + +if( $Elapsed ) { + if( -not $Status ) { + $progressParams.Status = "Running for $(ConvertFrom-TimeSpan $Elapsed)" + } + if( $TotalSeconds ) { + $progressParams.PercentComplete = [Math]::Floor($(1000 * (($Elapsed.TotalSeconds * 1.0) / ($TotalSeconds * 1.0)))/10.0) + } +} + +If( $ItemProgress ) { + if( $progressParams.Status ) { $progressParams.Status = " " + $progressParams.Status } + $progressParams.Status = "#$ItemProgress$(if($ItemCount){" out of $ItemCount"})" + $progressParams.Status +} + +$local:VerboseProgress = $("$([datetime]::Now.TimeOfDay.ToString("hh\:mm\:ss")) Progress: $($progressParams.Activity)" + + "$(if($progressParams.Status){" ($($progressParams.Status))"})" + + "$(if($ItemProgress) {" #$ItemProgress$(if($ItemCount){" out of $ItemCount"})"})" + + "$(if($progressParams.PercentComplete){" %$($progressParams.PercentComplete)"})" + + "$(if($progressParams.Completed){if(-not $progressParams.Status){" Done."}} else {"..."})" + ) + +Write-Verbose $VerboseProgress +if( Test-Path variable:global:psISE ) { + if( -not $VerbosePreference -eq 'Continue' ) { + Write-Host -ForegroundColor Cyan $VerboseProgress + } +} else { + Write-Progress @progressParams +} diff --git a/base/profile.d/PathProcessingFunctions.ps1.ps1 b/base/profile.d/PathProcessingFunctions.ps1.ps1 new file mode 100644 index 0000000..61840c5 --- /dev/null +++ b/base/profile.d/PathProcessingFunctions.ps1.ps1 @@ -0,0 +1,4 @@ +function Get-ShellSafePath { param([string]$LiteralPath) + "`"$([Management.Automation.WildcardPattern]::Escape($LiteralPath))`"" +} +