Version 1.4

added Get-PsOneToken and Expand-PsOneToken

PSOneTools/1.4/Expand-PSOneToken.ps1

@@ -0,0 +1,53 @@
+function Expand-PSOneToken
+{
+  <#
+      .SYNOPSIS
+      Expands all nested token from a token of type "StringExpandable"
+
+      .DESCRIPTION
+      Recursively emits all tokens embedded in a token of type "StringExpandable"
+      The original token is also emitted.
+
+      .EXAMPLE
+      Get-PSOneToken -Code '"Hello $host"' -TokenKind StringExpandable | Expand-PSOneToken 
+      Emits all tokens, including the embedded (nested) tokens
+      .LINK
+      https://powershell.one/powershell-internals/parsing-and-tokenization/advanced-tokenizer
+      https://github.com/TobiasPSP/Modules.PSOneTools/blob/master/PSOneTools/1.4/Expand-PSOneToken.ps1
+  #>
+
+  # use the most specific parameter as default:
+  [CmdletBinding(DefaultParameterSetName='ExpandableString')]
+  param
+  (
+    # binds a token of type "StringExpandableToken"
+    [Parameter(Mandatory,ParameterSetName='ExpandableString',
+                Position=0,ValueFromPipeline)]
+    [Management.Automation.Language.StringExpandableToken]
+    $StringExpandable,
+
+    # binds all tokens
+    [Parameter(Mandatory,ParameterSetName='Token',
+                Position=0,ValueFromPipeline)]
+    [Management.Automation.Language.Token]
+    $Token
+  )
+
+  process
+  {
+    switch($PSCmdlet.ParameterSetName)
+    {
+      # recursively expand token of type "StringExpandable"
+      'ExpandableString'  { 
+        $StringExpandable 
+        $StringExpandable.NestedTokens | 
+          Where-Object { $_ } | 
+          Expand-PSOneToken
+      }
+      # return regular token as-is:
+      'Token'             { $Token }
+      # should never occur:
+      default             { Write-Warning $_ }
+    }
+  }
+}

PSOneTools/1.4/Get-PSOneToken.ps1

@@ -0,0 +1,166 @@
+
+
+function Get-PSOneToken
+{
+  <#
+      .SYNOPSIS
+      Parses a PowerShell Script (*.ps1, *.psm1, *.psd1) and returns the token
+
+      .DESCRIPTION
+      Invokes the advanced PowerShell Parser and returns tokens and syntax errors
+
+      .EXAMPLE
+      Get-PSOneToken -Path c:\test.ps1
+      Parses the content of c:\test.ps1 and returns tokens and syntax errors
+
+      .EXAMPLE
+      Get-ChildItem -Path $home -Recurse -Include *.ps1,*.psm1,*.psd1 -File |
+      Get-PSOneToken |
+      Out-GridView
+
+      parses all PowerShell files found anywhere in your user profile
+
+      .EXAMPLE
+      Get-ChildItem -Path $home -Recurse -Include *.ps1,*.psm1,*.psd1 -File |
+      Get-PSOneToken |
+      Where-Object Errors
+
+      parses all PowerShell files found anywhere in your user profile
+      and returns only those files that contain syntax errors
+
+      .LINK
+      https://powershell.one/powershell-internals/parsing-and-tokenization/advanced-tokenizer
+      https://github.com/TobiasPSP/Modules.PSOneTools/blob/master/PSOneTools/1.4/Get-PSOneToken.ps1
+  #>
+
+  [CmdletBinding(DefaultParameterSetName='Path')]
+  param
+  (
+    # Path to PowerShell script file
+    # can be a string or any object that has a "Path" 
+    # or "FullName" property:
+    [String]
+    [Parameter(Mandatory,ValueFromPipeline,ParameterSetName='Path')]
+    [Alias('FullName')]
+    $Path,
+
+    # PowerShell Code as ScriptBlock
+    [ScriptBlock]
+    [Parameter(Mandatory,ValueFromPipeline,ParameterSetName='ScriptBlock')]
+    $ScriptBlock,
+
+
+    # PowerShell Code as String
+    [String]
+    [Parameter(Mandatory, ValueFromPipeline,ParameterSetName='Code')]
+    $Code,
+
+    # the kind of token requested. If neither TokenKind nor TokenFlag is requested, 
+    # a full tokenization occurs
+    [System.Management.Automation.Language.TokenKind[]]
+    $TokenKind = $null,
+
+    # the kind of token requested. If neither TokenKind nor TokenFlag is requested, 
+    # a full tokenization occurs
+    [System.Management.Automation.Language.TokenFlags[]]
+    $TokenFlag = $null,
+
+    # include nested token that are contained inside 
+    # ExpandableString tokens
+    [Switch]
+    $IncludeNestedToken
+
+  )
+
+  begin
+  {
+    # create variables to receive tokens and syntax errors:
+    $errors = 
+    $tokens = $null
+
+    # return tokens only?
+    # when the user submits either one of these parameters, the return value should
+    # be tokens of these kinds:
+    $returnTokens = ($PSBoundParameters.ContainsKey('TokenKind')) -or 
+    ($PSBoundParameters.ContainsKey('TokenFlag'))
+  }
+  process
+  {
+    # if a scriptblock was submitted, convert it to string
+    if ($PSCmdlet.ParameterSetName -eq 'ScriptBlock')
+    {
+      $Code = $ScriptBlock.ToString()
+    }
+
+    # if a path was submitted, read code from file,
+    if ($PSCmdlet.ParameterSetName -eq 'Path')
+    {
+      $code = Get-Content -Path $Path -Raw -Encoding Default
+      $name = Split-Path -Path $Path -Leaf
+      $filepath = $Path
+
+      # parse the file:
+      $ast = [System.Management.Automation.Language.Parser]::ParseFile(
+        $Path, 
+        [ref] $tokens, 
+      [ref]$errors)
+    }
+    else
+    {
+      # else the code is already present in $Code
+      $name = $Code
+      $filepath = ''
+
+      # parse the string code:
+      $ast = [System.Management.Automation.Language.Parser]::ParseInput(
+        $Code, 
+        [ref] $tokens, 
+      [ref]$errors)
+    }
+
+    if ($IncludeNestedToken)
+    {
+      # "unwrap" nested token
+      $tokens = $tokens | Expand-PSOneToken
+    }
+
+    if ($returnTokens)
+    {
+      # filter token and use fast scriptblock filtering instead of Where-Object:
+      $tokens |
+      & { process { if ($TokenKind -eq $null -or 
+          $TokenKind -contains $_.Kind) 
+          { $_ }
+      }} |
+      & { process {
+          $token = $_
+          if ($TokenFlag -eq $null) { $token }
+          else {
+            $TokenFlag | 
+            Foreach-Object { 
+              if ($token.TokenFlags.HasFlag($_)) 
+            { $token } } | 
+            Select-Object -First 1
+          }
+        }
+      }
+
+    }
+    else
+    {
+      # return the results as a custom object
+      [PSCustomObject]@{
+        Name = $name
+        Path = $filepath
+        Tokens = $tokens
+        # "move" nested "Extent" up one level 
+        # so all important properties are shown immediately
+        Errors = $errors | 
+        Select-Object -Property Message, 
+        IncompleteInput, 
+        ErrorId -ExpandProperty Extent
+        Ast = $ast
+      }
+    }  
+  }
+}

PSOneTools/1.3/Test-PSOneScript.ps1 → PSOneTools/1.4/Test-PSOneScript.ps1

@@ -29,19 +29,29 @@ function Test-PSOneScript
       and returns only those files that contain syntax errors
 
       .LINK
-      https://powershell.one
-  #>
+      https://powershell.one/powershell-internals/parsing-and-tokenization/simple-tokenizer
+      https://github.com/TobiasPSP/Modules.PSOneTools/blob/master/PSOneTools/1.4/Test-PSOneScript.ps1
 
+  #>
 
+  [CmdletBinding(DefaultParameterSetName='Path')]
   param
   (
     # Path to PowerShell script file
     # can be a string or any object that has a "Path" 
     # or "FullName" property:
     [String]
-    [Parameter(Mandatory,ValueFromPipeline)]
+    [Parameter(Mandatory,ValueFromPipeline,ParameterSetName='Path')]
     [Alias('FullName')]
-    $Path
+    $Path,
+
+    # PowerShell Code as String
+    # you can also submit a ScriptBlock which will automatically be converted
+    # to a string. ScriptBlocks by default cannot contain syntax errors because
+    # they are parsed already.
+    [String]
+    [Parameter(Mandatory,ValueFromPipeline,ParameterSetName='Code')]
+    $Code
   )
 
   begin
@@ -53,12 +63,25 @@ function Test-PSOneScript
     # create a variable to receive syntax errors:
     $errors = $null
     # tokenize PowerShell code:
-    $code = Get-Content -Path $Path -Raw -Encoding Default
-
+
+    # if a path was submitted, read code from file,
+    if ($PSCmdlet.ParameterSetName -eq 'Path')
+    {
+        $code = Get-Content -Path $Path -Raw -Encoding Default
+        $name = Split-Path -Path $Path -Leaf
+        $filepath = $Path
+    }
+    else
+    {
+        # else the code is already present in $Code
+        $name = $Code
+        $filepath = ''
+    }
+
     # return the results as a custom object
     [PSCustomObject]@{
-      Name = Split-Path -Path $Path -Leaf
-      Path = $Path
+      Name = $name
+      Path = $filepath
       Tokens = [Management.Automation.PSParser]::Tokenize($code, [ref]$errors)
       Errors = $errors | Select-Object -ExpandProperty Token -Property Message
     }  

PSOneTools/1.3/module.psm1 → PSOneTools/1.4/module.psm1

@@ -13,4 +13,8 @@
 . $PSScriptRoot\Foreach-ObjectFast.ps1
 . $PSScriptRoot\Where-ObjectFast.ps1
 . $PSScriptRoot\Test-PSOneScript.ps1
+. $PSScriptRoot\Get-PSOneToken.ps1
+. $PSScriptRoot\Expand-PSOneToken.ps1
+
+