Set Up Cloud Discovery & Shadow IT Analysis

# ---------------------------------------------------------------
# PURPOSE: Verify connectivity to the Defender for Cloud Apps API.
# WHY: Before running any automation, confirm your API token is valid
#      and the MDA endpoint is reachable. A 401 means an invalid token;
#      a timeout means a firewall or DNS issue.
# PREREQUISITES: Generate an API token in Settings > Cloud Apps > API tokens.
#      Replace YOUR_API_TOKEN_HERE with your actual token.
# OUTPUT: On success → "Connected" + total discovered app count.
#         On failure → the HTTP error message (401, 403, timeout, etc.).
# ---------------------------------------------------------------
$mdaUrl = "https://contoso.us3.portal.cloudappsecurity.com/api/v1/discovery/"
$token  = "YOUR_API_TOKEN_HERE"
$headers = @{
    "Authorization" = "Token $token"   # MDA uses token-based auth (not Bearer)
    "Content-Type"  = "application/json"
}

# Send a simple GET to the discovered_apps endpoint as a connectivity check.
# TimeoutSec 15 avoids hanging if outbound 443 is blocked.
try {
    $response = Invoke-RestMethod -Uri "$($mdaUrl)discovered_apps/" `
        -Headers $headers -Method GET -TimeoutSec 15
    Write-Host "Connected to MDA API. Discovered apps count: $($response.total)" -ForegroundColor Green
} catch {
    Write-Host "Connection failed: $($_.Exception.Message)" -ForegroundColor Red
}

# ---------------------------------------------------------------
# PURPOSE: Export outbound traffic logs from a Palo Alto firewall
#          for upload to MDA Cloud Discovery as a snapshot report.
# WHY: MDA needs raw traffic logs to identify which cloud apps
#      your users are accessing. Palo Alto logs contain source IP,
#      destination URL, bytes, and action - exactly what MDA parses.
# ---------------------------------------------------------------

# OPTION A - Palo Alto CLI (SSH session):
# > set cli pager off          ← disables paged output
# > show log traffic direction equal 1 receive_time in last-30-days
# direction=1 means outbound traffic only. Save the output as CSV.

# OPTION B - Palo Alto XML API (programmatic export):
# Replace YOUR_API_KEY with the key from Device > API Keys on the firewall.
$paloAltoHost = "https://pa-firewall.contoso.com"
$apiKey = "YOUR_API_KEY"

# Query filter: outbound traffic (direction eq 1) from Jan 1 2026 onward.
# Adjust the date range to capture 7–30 days of traffic for a meaningful snapshot.
$logQuery = "( direction eq 1 ) and ( receive_time geq '2026/01/01 00:00:00' )"

# Call the Palo Alto REST API to retrieve matching traffic log entries.
$uri = "$paloAltoHost/api/?type=log&log-type=traffic&query=$logQuery&key=$apiKey"
$logData = Invoke-RestMethod -Uri $uri -Method GET

# Save to disk in UTF-8. MDA accepts CSV, TSV, syslog, W3C, and CEF formats.
$logData | Out-File -FilePath "C:\Logs\paloalto-traffic-export.csv" -Encoding UTF8

# OUTPUT: File size in MB - if >5 GB, split before uploading to the MDA portal.
Write-Host "Log file exported. Size: $((Get-Item 'C:\Logs\paloalto-traffic-export.csv').Length / 1MB) MB"

# ---------------------------------------------------------------
# PURPOSE: Retrieve discovered apps from MDA sorted by risk score
#          (ascending) so the riskiest apps appear first.
# WHY: After a snapshot or continuous report processes, you need to
#      identify which shadow IT apps pose the greatest risk. Sorting
#      by ascending risk (1=highest risk, 10=lowest) surfaces the
#      most dangerous apps at the top for immediate triage.
# OUTPUT: A table of the 20 riskiest apps showing name, risk score,
#         category (e.g. Cloud Storage, CRM), user count, and
#         traffic volume in MB. High user counts with low risk scores
#         are your top priority for remediation.
# ---------------------------------------------------------------
$mdaUrl  = "https://contoso.us3.portal.cloudappsecurity.com/api/v1/"
$token   = "YOUR_API_TOKEN_HERE"
$headers = @{
    "Authorization" = "Token $token"
    "Content-Type"  = "application/json"
}

# POST request with sort parameters. limit=50 fetches the top 50;
# change to 500+ for a full catalogue export.
$body = @{
    filters = @{}                # No filter = all apps
    sortField = "risk"           # Sort by risk score
    sortDirection = "asc"        # Ascending: riskiest (score 1) first
    limit = 50
} | ConvertTo-Json -Depth 5

$apps = Invoke-RestMethod -Uri "$($mdaUrl)discovered_apps/" `
    -Headers $headers -Method POST -Body $body

# Format the top 20 into a readable table.
# Traffic is converted from bytes to megabytes for readability.
$apps.data | Select-Object -First 20 | ForEach-Object {
    [PSCustomObject]@{
        AppName   = $_.name        # Display name of the cloud app
        RiskScore = $_.risk        # 1 (highest risk) to 10 (lowest risk)
        Category  = $_.category    # Cloud Storage, Collaboration, CRM, etc.
        Users     = $_.users       # Number of unique users accessing this app
        Traffic   = [math]::Round($_.traffic / 1MB, 2)  # Data volume in MB
    }
} | Format-Table -AutoSize

Write-Host "`nTotal Discovered Apps: $($apps.total)" -ForegroundColor Cyan

# ---------------------------------------------------------------
# PURPOSE: Export all discovered apps to CSV after adjusting score
#          metrics, so you can compare before/after risk scores and
#          share with stakeholders for governance decisions.
# WHY: When you change scoring weights (e.g. increase Security to 10),
#      risk scores recalculate. Exporting to CSV lets you track
#      score changes, share with compliance/legal, and document
#      which apps became higher/lower risk after customisation.
# OUTPUT: CSV file at C:\Reports\ with columns: AppName, RiskScore,
#         Category, Users, TrafficMB, ComplianceRisk, SecurityRisk.
#         Open in Excel to sort/filter by any column.
# ---------------------------------------------------------------
$mdaUrl  = "https://contoso.us3.portal.cloudappsecurity.com/api/v1/"
$token   = "YOUR_API_TOKEN_HERE"
$headers = @{
    "Authorization" = "Token $token"
    "Content-Type"  = "application/json"
}

# Fetch up to 500 apps sorted by risk. Increase limit for larger environments.
$body = @{
    filters = @{}
    sortField = "risk"
    sortDirection = "asc"
    limit = 500
} | ConvertTo-Json -Depth 5

$apps = Invoke-RestMethod -Uri "$($mdaUrl)discovered_apps/" `
    -Headers $headers -Method POST -Body $body

# Build a flat object for each app and export to CSV.
# ComplianceRisk/SecurityRisk show the sub-score breakdown so
# stakeholders can see exactly why an app's overall score changed.
$apps.data | ForEach-Object {
    [PSCustomObject]@{
        AppName        = $_.name             # Cloud app display name
        RiskScore      = $_.risk             # Overall risk score (1–10)
        Category       = $_.category         # E.g. Cloud Storage, CRM
        Users          = $_.users            # Unique users accessing the app
        TrafficMB      = [math]::Round($_.traffic / 1MB, 2)  # Traffic in MB
        ComplianceRisk = $_.complianceRisk   # Compliance sub-score
        SecurityRisk   = $_.securityRisk     # Security sub-score
    }
} | Export-Csv -Path "C:\Reports\MDA-DiscoveredApps-Scored.csv" -NoTypeInformation

Write-Host "Exported $($apps.total) apps to C:\Reports\MDA-DiscoveredApps-Scored.csv" -ForegroundColor Green

# ---------------------------------------------------------------
# PURPOSE: Deploy the MDA Docker-based log collector that receives
#          syslog from your firewall and uploads logs to the cloud.
# WHY: Snapshot reports are one-time; a log collector provides
#      continuous, automated Cloud Discovery - new apps are detected
#      within hours of first use, not weeks later.
# PREREQUISITES:
#   - Linux VM (Ubuntu 20.04+), 2 CPU, 4 GB RAM, 100 GB disk
#   - Docker Engine installed
#   - Outbound HTTPS (443) to *.portal.cloudappsecurity.com
#   - Data source & log collector created in the MDA portal
# ---------------------------------------------------------------

# Step 1: Install Docker on the Linux VM (if not already installed)
# sudo apt-get update
# sudo apt-get install -y docker.io
# sudo systemctl enable docker --now

# Step 2: Run the MDA Log Collector container.
# Replace PUBLICIP, CONSOLE, COLLECTOR, and TOKEN with values
# from Settings > Cloud Apps > Automatic log upload.
$dockerCommand = @"
docker run -d --name mda-log-collector \
  --restart always \
  -p 514:514/udp \
  -p 514:514/tcp \
  -p 21:21/tcp \
  -p 20000-20099:20000-20099/tcp \
  -e "PUBLICIP=10.0.1.50" \
  -e "PROXY=" \
  -e "CONSOLE=contoso.us3.portal.cloudappsecurity.com" \
  -e "COLLECTOR=ContosoLogCollector01" \
  -e "TOKEN=YOUR_COLLECTOR_TOKEN_HERE" \
  mcr.microsoft.com/mcas/logcollector:latest
"@
# Flags explained:
#   -d              → run in detached (background) mode
#   --restart always → auto-restart on crash or host reboot
#   -p 514:514/udp  → receive syslog over UDP (most firewalls)
#   -p 514:514/tcp  → receive syslog over TCP (for reliable delivery)
#   -p 21 + 20000+  → FTP ports for firewalls that send logs via FTP
#   PUBLICIP        → the IP your firewall sends syslog to
#   CONSOLE         → your MDA regional endpoint URL
#   COLLECTOR       → the collector name you defined in the portal
#   TOKEN           → authentication token from the portal

Write-Host $dockerCommand -ForegroundColor Cyan
Write-Host "`n# Copy and run this command on your Docker host" -ForegroundColor Yellow

# Step 3: Verify the container is running and check logs for errors.
# docker ps --filter "name=mda-log-collector"
# docker logs mda-log-collector --tail 50
# Look for "Connected to console" in the output.

# Step 4: Test syslog reception - send a dummy syslog message locally.
# If the collector is working, this line appears in docker logs.
# echo "<134>Jan 01 00:00:00 test-fw traffic: allow src=10.0.0.5 dst=example.com" | nc -u 127.0.0.1 514

# ---------------------------------------------------------------
# PURPOSE: Check the health of all deployed log collectors via API.
# WHY: A disconnected collector means a gap in Cloud Discovery data.
#      Monitor status regularly to catch issues before they create
#      blind spots (e.g. expired certificates, disk full, network change).
# OUTPUT: Table showing each collector's Name, Status (Connected/
#         Disconnected), LastUpdated timestamp, and assigned DataSources.
#         Status = "Connected" + recent LastUpdated = healthy.
# ---------------------------------------------------------------
$mdaUrl  = "https://contoso.us3.portal.cloudappsecurity.com/api/v1/"
$token   = "YOUR_API_TOKEN_HERE"
$headers = @{
    "Authorization" = "Token $token"
    "Content-Type"  = "application/json"
}

# GET request to the log_collectors endpoint returns all registered collectors.
$collectors = Invoke-RestMethod -Uri "$($mdaUrl)discovery/log_collectors/" `
    -Headers $headers -Method GET

# Format the response into a readable status table.
$collectors.data | ForEach-Object {
    [PSCustomObject]@{
        Name        = $_.name            # Collector display name
        Status      = $_.status          # Connected / Disconnected / Warning
        LastUpdated = $_.lastModified     # Last time logs were received
        DataSources = ($_.dataSources | ForEach-Object { $_.name }) -join ", "
    }
} | Format-Table -AutoSize

# ---------------------------------------------------------------
# PURPOSE: Verify that MDE → MDA integration is enabled and that
#          endpoints are reporting cloud app data to Cloud Discovery.
# WHY: MDE integration is the most powerful discovery source - it
#      captures cloud app usage from every onboarded endpoint, even
#      when users are off-network (home Wi-Fi, VPN, cellular).
#      This script checks both the integration toggle and the custom
#      network indicators feature (required for blocking in Step 9).
# OUTPUT: Green checkmarks for enabled features; red/yellow for
#         disabled features that need attention. Also shows the
#         count of MDE endpoints actively reporting to Cloud Discovery.
# ---------------------------------------------------------------

# Use an MDE API token (Bearer auth, not Token auth like MDA).
# Generate at security.microsoft.com > Settings > APIs > API tokens.
$mdeToken = "YOUR_MDE_API_TOKEN"
$headers = @{
    "Authorization" = "Bearer $mdeToken"
    "Content-Type"  = "application/json"
}

# Check if MDA integration feature is enabled
try {
    $features = Invoke-RestMethod -Uri "https://api.securitycenter.microsoft.com/api/preferences" `
        -Headers $headers -Method GET
    
    $mdaFeature = $features | Where-Object { $_.featureName -eq "cloudAppsIntegration" }
    if ($mdaFeature.isEnabled) {
        Write-Host "? MDE → MDA integration is ENABLED" -ForegroundColor Green
    } else {
        Write-Host "? MDE → MDA integration is DISABLED" -ForegroundColor Red
    }
    
    $networkIndicators = $features | Where-Object { $_.featureName -eq "customNetworkIndicators" }
    if ($networkIndicators.isEnabled) {
        Write-Host "? Custom network indicators are ENABLED (required for app blocking)" -ForegroundColor Green
    } else {
        Write-Host "? Custom network indicators are DISABLED · enable for app blocking" -ForegroundColor Yellow
    }
} catch {
    Write-Host "Error querying MDE API: $($_.Exception.Message)" -ForegroundColor Red
}

# Check how many MDE-onboarded devices are reporting cloud app data
$mdaUrl  = "https://contoso.us3.portal.cloudappsecurity.com/api/v1/"
$mdaToken = "YOUR_MDA_API_TOKEN"
$mdaHeaders = @{
    "Authorization" = "Token $mdaToken"
    "Content-Type"  = "application/json"
}

$devices = Invoke-RestMethod -Uri "$($mdaUrl)discovery/discovered_apps/?source=mde" `
    -Headers $mdaHeaders -Method GET

Write-Host "`nMDE endpoints reporting to Cloud Discovery: $($devices.total)" -ForegroundColor Cyan

# ---------------------------------------------------------------
# PURPOSE: Bulk-tag all discovered apps with risk score 1–3 as
#          "Unsanctioned" using the MDA API.
# WHY: Apps scoring 1–3 have significant security, compliance, or
#      legal deficiencies (e.g. no encryption, no SOC 2, poor privacy
#      policy). Tagging them as Unsanctioned prepares them for
#      endpoint-level blocking via MDE (Step 9) and marks them
#      in the portal as unapproved for your organisation.
# OUTPUT: Per-app confirmation of tagging, plus a total count.
#         If MDE blocking is enabled, unsanctioned app URLs will be
#         pushed as network indicators within ~2 hours.
# ---------------------------------------------------------------
$mdaUrl  = "https://contoso.us3.portal.cloudappsecurity.com/api/v1/"
$token   = "YOUR_API_TOKEN_HERE"
$headers = @{
    "Authorization" = "Token $token"
    "Content-Type"  = "application/json"
}

# Filter: risk score between 1 (highest risk) and 3.
# Adjust the range to match your organisation's risk threshold.
$body = @{
    filters = @{
        risk = @{ gte = 1; lte = 3 }
    }
    limit = 200
} | ConvertTo-Json -Depth 5

$riskyApps = Invoke-RestMethod -Uri "$($mdaUrl)discovered_apps/" `
    -Headers $headers -Method POST -Body $body

Write-Host "Found $($riskyApps.total) high-risk apps (score 1-3)" -ForegroundColor Yellow

# Loop through each app and apply the "unsanctioned" tag via the tag endpoint.
foreach ($app in $riskyApps.data) {
    $tagBody = @{ appId = $app.appId; tag = "unsanctioned" } | ConvertTo-Json
    try {
        Invoke-RestMethod -Uri "$($mdaUrl)discovered_apps/tag/" `
            -Headers $headers -Method POST -Body $tagBody
        Write-Host "  Tagged '$($app.name)' (Score: $($app.risk)) as Unsanctioned" -ForegroundColor Red
    } catch {
        Write-Host "  Failed to tag '$($app.name)': $($_.Exception.Message)" -ForegroundColor DarkRed
    }
}

Write-Host "`nBulk tagging complete." -ForegroundColor Green

# ---------------------------------------------------------------
# PURPOSE: List all Cloud Discovery policies currently configured
#          in your MDA tenant via the API.
# WHY: Auditing your discovery policies ensures you have coverage
#      for new apps, risky apps, and anomalous usage patterns.
#      This helps verify that the policies created in Step 8 are
#      active and correctly configured.
# OUTPUT: Table of policies showing name, severity level, enabled
#         status, and creation date. Disabled policies appear but
#         are not actively triggering alerts.
# ---------------------------------------------------------------
$mdaUrl  = "https://contoso.us3.portal.cloudappsecurity.com/api/v1/"
$token   = "YOUR_API_TOKEN_HERE"
$headers = @{
    "Authorization" = "Token $token"
    "Content-Type"  = "application/json"
}

# Filter by policyType = 5, which corresponds to Cloud Discovery policies.
# Other policy types: 0=Activity, 1=File, 2=OAuth app, etc.
$body = @{
    filters = @{
        policyType = @{ eq = 5 }  # 5 = Discovery policy type
    }
} | ConvertTo-Json -Depth 5

$policies = Invoke-RestMethod -Uri "$($mdaUrl)policies/" `
    -Headers $headers -Method POST -Body $body

$policies.data | ForEach-Object {
    [PSCustomObject]@{
        PolicyName = $_.name       # Policy display name
        Severity   = $_.severity   # Low, Medium, High
        Enabled    = $_.enabled    # True = actively monitoring
        Created    = $_.created    # Creation timestamp
    }
} | Format-Table -AutoSize

Write-Host "Total Discovery policies: $($policies.total)" -ForegroundColor Cyan

# ---------------------------------------------------------------
# PURPOSE: Confirm that MDA-unsanctioned app URLs have been pushed
#          to MDE as custom network indicators (block rules).
# WHY: When you tag an app as "Unsanctioned" and enable MDE blocking,
#      MDA automatically creates URL indicators in MDE. This script
#      verifies those indicators exist and tests endpoint blocking.
# OUTPUT: Table of blocked URLs with action (Block/Allow), severity,
#         and alert settings. The test section confirms end-to-end
#         blocking by attempting to reach a blocked domain.
# ---------------------------------------------------------------
$mdeToken = "YOUR_MDE_API_TOKEN"
$headers = @{
    "Authorization" = "Bearer $mdeToken"
    "Content-Type"  = "application/json"
}

# Query the MDE indicators API, filtering for indicators created
# by the CloudAppSecurity source (i.e. MDA integration).
$indicators = Invoke-RestMethod `
    -Uri "https://api.securitycenter.microsoft.com/api/indicators?`$filter=source eq 'CloudAppSecurity'" `
    -Headers $headers -Method GET

Write-Host "MDA-created network indicators (blocked apps):" -ForegroundColor Cyan
$indicators.value | Select-Object -First 20 | ForEach-Object {
    [PSCustomObject]@{
        URL          = $_.indicatorValue   # The blocked URL or domain
        Action       = $_.action           # Block, Warn, or Allow
        Severity     = $_.severity         # Informational, Low, Medium, High
        Title        = $_.title            # Description of the indicator
        GenerateAlert = $_.generateAlert   # Whether alerts are raised on access
    }
} | Format-Table -AutoSize

Write-Host "`nTotal blocked indicators: $($indicators.value.Count)" -ForegroundColor Yellow

# --- Endpoint Blocking Test ---
# Run this section FROM an MDE-onboarded endpoint (not a server).
# A successful block causes a connection error; HTTP 200 means NOT blocked.
$testUrl = "https://blocked-app-example.com"
try {
    $result = Invoke-WebRequest -Uri $testUrl -TimeoutSec 5 -UseBasicParsing
    Write-Host "WARNING: $testUrl was NOT blocked (Status: $($result.StatusCode))" -ForegroundColor Red
} catch {
    Write-Host "CONFIRMED: $testUrl is blocked by MDE network protection" -ForegroundColor Green
}

# ---------------------------------------------------------------
# PURPOSE: Generate a comprehensive Shadow IT executive report as
#          both a CSV file and a console summary dashboard.
# WHY: Leadership needs clear metrics to understand the organisation's
#      shadow IT posture: total apps, governance status, risk
#      distribution, and data volumes. This report serves as the
#      monthly deliverable for the Cloud App Governance Owner.
# OUTPUT:
#   1. CSV file with per-app details (name, risk, tag, users, traffic,
#      compliance certs) - suitable for Excel pivot tables.
#   2. Console summary showing total/sanctioned/unsanctioned/high-risk
#      counts - useful for quick status checks.
# ---------------------------------------------------------------

$mdaUrl  = "https://contoso.us3.portal.cloudappsecurity.com/api/v1/"
$token   = "YOUR_API_TOKEN_HERE"
$headers = @{
    "Authorization" = "Token $token"
    "Content-Type"  = "application/json"
}

# Fetch up to 1000 apps sorted by traffic (highest volume first).
# Sorting by traffic highlights apps consuming the most bandwidth.
$body = @{
    filters = @{}
    sortField = "traffic"
    sortDirection = "desc"
    limit = 1000
} | ConvertTo-Json -Depth 5

$allApps = Invoke-RestMethod -Uri "$($mdaUrl)discovered_apps/" `
    -Headers $headers -Method POST -Body $body

# Build a date-stamped report path.
$reportDate = Get-Date -Format "yyyy-MM-dd"
$reportPath = "C:\Reports\ShadowIT-Executive-Report-$reportDate.csv"

# Transform each app into a flat row for the CSV.
$report = $allApps.data | ForEach-Object {
    [PSCustomObject]@{
        "App Name"       = $_.name                  # Cloud app display name
        "Category"       = $_.category               # E.g. Cloud Storage, AI
        "Risk Score"     = $_.risk                    # 1 (worst) to 10 (best)
        "Tag"            = $_.tag                     # Sanctioned / Unsanctioned / Under Review
        "Users"          = $_.users                   # Unique user count
        "Traffic (MB)"   = [math]::Round($_.traffic / 1MB, 2)  # Data volume
        "Transactions"   = $_.transactions            # Total API/web requests
        "Headquarters"   = $_.headquarters            # App vendor's HQ country
        "Compliance"     = ($_.complianceCertifications -join "; ")  # SOC 2, ISO, etc.
    }
}

$report | Export-Csv -Path $reportPath -NoTypeInformation -Encoding UTF8

# Calculate governance summary statistics for the console dashboard.
$totalApps    = $allApps.total
$sanctioned   = ($allApps.data | Where-Object { $_.tag -eq "sanctioned" }).Count
$unsanctioned = ($allApps.data | Where-Object { $_.tag -eq "unsanctioned" }).Count
$underReview  = ($allApps.data | Where-Object { $_.tag -eq "under review" }).Count
$highRisk     = ($allApps.data | Where-Object { $_.risk -le 4 }).Count

Write-Host "----------------------------------------" -ForegroundColor Cyan
Write-Host "   SHADOW IT EXECUTIVE SUMMARY" -ForegroundColor Cyan
Write-Host "----------------------------------------" -ForegroundColor Cyan
Write-Host "  Report Date:        $reportDate"
Write-Host "  Total Discovered:   $totalApps apps"
Write-Host "  Sanctioned:         $sanctioned apps"
Write-Host "  Unsanctioned:       $unsanctioned apps"
Write-Host "  Under Review:       $underReview apps"
Write-Host "  High Risk (=4):     $highRisk apps"
Write-Host "----------------------------------------" -ForegroundColor Cyan
Write-Host "`nDetailed report saved to: $reportPath" -ForegroundColor Green