package PVE::Storage::Custom::PureStoragePlugin; use strict; use warnings; use Data::Dumper qw( Dumper ); # DEBUG use IO::File (); use File::Path (); use PVE::JSONSchema (); use PVE::Network (); use PVE::Tools qw( file_read_firstline run_command ); use PVE::INotify (); use PVE::Storage::Plugin (); use JSON::XS qw( decode_json encode_json ); use LWP::UserAgent (); use HTTP::Headers (); use HTTP::Request (); use URI::Escape qw( uri_escape ); use File::Basename qw( basename ); use Time::HiRes qw( gettimeofday sleep ); use Cwd qw( abs_path ); use base qw(PVE::Storage::Plugin); push @PVE::Storage::Plugin::SHARED_STORAGE, 'purestorage'; $Data::Dumper::Terse = 1; # Removes `$VAR1 =` in output $Data::Dumper::Indent = 1; # Outputs everything in one line $Data::Dumper::Useqq = 1; # Uses quotes for strings # Error code constants for API requests use constant { ERROR_TOKEN_UPDATED => -1, # Token was refreshed (success, but need to update cache) ERROR_SUCCESS => 0, # Request succeeded ERROR_API_ERROR => 1, # PureStorage API returned an error ERROR_NETWORK_ERROR => 2, # Network or connectivity error ERROR_AUTH_FAILED => 3, # Authentication failed }; # Token state constants for authentication state machine use constant { TOKEN_STATE_LOGIN => 0, # Performing login request (using api-token) TOKEN_STATE_NEEDED => 1, # Need to obtain session token TOKEN_STATE_CACHED => 2, # Have valid cached session token }; my $PSFA_API = '2.26'; my $purestorage_wwn_prefix = '3624a9370'; my $default_hgsuffix = ""; my $default_protocol = 'iscsi'; # Global debug level (can be overridden per-storage or via environment) my $DEBUG = $ENV{ PURESTORAGE_DEBUG } // 0; # Get effective debug level for a storage config sub get_debug_level { my ( $scfg ) = @_; return $scfg->{ debug } if defined $scfg && defined $scfg->{ debug }; return $DEBUG; } # Set debug level from storage config (updates global $DEBUG) sub set_debug_from_config { my ( $scfg ) = @_; if ( defined $scfg && defined $scfg->{ debug } ) { $DEBUG = $scfg->{ debug }; } } ### BLOCK: Configuration sub api { # PVE 5: APIVER 2 # PVE 6: APIVER 3 # PVE 6: APIVER 4 e6f4eed43581de9b9706cc2263c9631ea2abfc1a / volume_has_feature # PVE 6: APIVER 5 a97d3ee49f21a61d3df10d196140c95dde45ec27 / allow rename # PVE 6: APIVER 6 8f26b3910d7e5149bfa495c3df9c44242af989d5 / prune_backups (fine, we don't support that content type) # PVE 6: APIVER 7 2c036838ed1747dabee1d2c79621c7d398d24c50 / volume_snapshot_needs_fsfreeze (guess we are fine, upstream only implemented it for RDBPlugin; we are not that different to let's say LVM in this regard) # PVE 6: APIVER 8 343ca2570c3972f0fa1086b020bc9ab731f27b11 / prune_backups (fine again, see APIVER 6) # PVE 7: APIVER 9 3cc29a0487b5c11592bf8b16e96134b5cb613237 / resets APIAGE! changes volume_import/volume_import_formats # PVE 7.1: APIVER 10 a799f7529b9c4430fee13e5b939fe3723b650766 / rm/add volume_snapshot_{list,info} (not used); blockers to volume_rollback_is_possible (not used) # PVE 8.4: APIVER 11 e2dc01ac9f06fe37cf434bad9157a50ecc4a99ce / new_backup_provider/sensitive_properties; backup provider might be interesting, we can look at it later # PVE 9: APIVER 12 280bb6be777abdccd89b1b1d7bdd4feaba9af4c2 / qemu_blockdev_options/rename_snapshot/get_formats my $tested_apiver = 12; my $apiver = PVE::Storage::APIVER; my $apiage = PVE::Storage::APIAGE; # the plugin supports multiple PVE generations, currently we did not break anything, tell them what they want to hear if possible if ( $apiver >= 2 and $apiver <= $tested_apiver ) { return $apiver; } # if we are still in the APIAGE, we can still report what we have if ( $apiver - $apiage < $tested_apiver ) { return $tested_apiver; } # lowest apiver we support return 10; } sub type { return "purestorage"; } sub plugindata { return { content => [ { images => 1, none => 1 }, { images => 1 } ], format => [ { raw => 1 }, "raw" ], }; } sub properties { return { hgsuffix => { description => "Host group suffx.", type => 'string', default => $default_hgsuffix }, address => { description => "PureStorage Management IP address or DNS name.", type => 'string' }, token => { description => "Storage API token.", type => 'string' }, podname => { description => "PureStorage pod name", type => 'string' }, vnprefix => { description => "Prefix to add to volume name before sending it to PureStorage array", type => 'string' }, check_ssl => { description => "Verify the server's TLS certificate", type => 'boolean', default => 'no' }, protocol => { description => "Set storage protocol ( iscsi | fc | nvme )", type => 'string', default => $default_protocol }, token_ttl => { description => "Session token time-to-live in seconds.", type => 'integer', default => 3600 # Max 10h }, debug => { description => "Enable debug logging (0=off, 1=basic, 2=verbose, 3=trace).", type => 'integer', minimum => 0, maximum => 3, default => 0 }, }; } sub options { return { address => { fixed => 1 }, token => { fixed => 1 }, hgsuffix => { optional => 1 }, vgname => { optional => 1 }, podname => { optional => 1 }, vnprefix => { optional => 1 }, check_ssl => { optional => 1 }, protocol => { optional => 1 }, token_ttl => { optional => 1 }, debug => { optional => 1 }, nodes => { optional => 1 }, disable => { optional => 1 }, content => { optional => 1 }, format => { optional => 1 }, }; } ### BLOCK: Supporting functions my $cmd = { # fuser => '/usr/bin/fuser', multipath => '/sbin/multipath', multipathd => '/sbin/multipathd', blockdev => '/usr/sbin/blockdev', dmsetup => '/sbin/dmsetup', kpartx => '/sbin/kpartx', udevadm => '/usr/bin/udevadm', sync => '/usr/bin/sync' }; # Get full path for a command, checking availability sub get_command_path { my ( $name ) = @_; # Check all commands on first use ensure_commands_checked(); my $path = $cmd->{ $name }; if ( !defined $path ) { die "Error :: Unknown command '$name'\n"; } if ( !-x $path ) { die "Error :: Command '$name' not found or not executable at '$path'\n"; } return $path; } # Check if required commands are available on the system sub check_commands { my @missing; foreach my $name ( keys %$cmd ) { my $path = $cmd->{ $name }; if ( !-x $path ) { push @missing, "$name ($path)"; } } if ( @missing ) { warn "Warning :: The following commands are not available or not executable:\n"; warn " - $_\n" foreach @missing; warn "Plugin functionality may be limited.\n"; } return scalar @missing == 0; } # Check commands availability - called lazily on first use my $commands_checked = 0; sub ensure_commands_checked { return if $commands_checked; check_commands(); $commands_checked = 1; } sub exec_command { my ( $command, $dm, %param ) = @_; $dm //= 1; # Try to resolve command path if it's a known command name my $cmd_name = $command->[0]; if ( exists $cmd->{ $cmd_name } ) { eval { $command->[0] = get_command_path( $cmd_name ); }; if ( $@ ) { # Command not available, but continue with original name # This allows system PATH resolution as fallback warn "Warning :: $@" if $dm >= 0; } } print "Debug :: execute '" . join( ' ', @$command ) . "'\n" if $DEBUG >= 2; if ( $DEBUG < 3 ) { $param{ 'quiet' } = 1 unless exists $param{ 'quiet' }; } eval { run_command( $command, %param ) }; if ( $@ ) { my $error = " :: Cannot execute '" . join( ' ', @$command ) . "'\n ==> Error :: $@\n"; die 'Error' . $error if $dm > 0; warn 'Warning' . $error unless $dm < 0; return $dm < 0; } return $dm >= 0; } sub scsi_scan_new { my ( $protocol ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::scsi_scan_new\n" if $DEBUG; my $path = '/sys/class/' . $protocol . '_host'; opendir( my $dh, $path ) or die "Cannot open directory: $!"; my @hosts = grep { !/^\.\.?$/ } readdir( $dh ); closedir( $dh ); my $count = 0; foreach my $host ( @hosts ) { next unless $host =~ /^(\w+)$/; $path = '/sys/class/scsi_host/' . $1; # untaint if ( -d $path ) { device_op( $path, 'scan', '- - -' ); ++$count; } else { warn "Warning :: SCSI host path $path does not exist.\n"; } } die "Error :: Did not find hosts to scan.\n" unless $count > 0; print "Debug :: Scanned $count host" . ( $count > 1 ? 's' : '' ) . " for new devices\n" if $DEBUG; } sub multipath_check { my ( $wwid ) = @_; # TODO: Find a better check # TODO: Support non-multipath mode my $multipath_cmd = get_command_path( 'multipath' ); my $output = `$multipath_cmd -l $wwid 2>/dev/null`; return $output ne ''; } sub wait_for { my ( $success, $message, $timeout, $delay ) = @_; my $debug = 'Debug :: Waiting for ' . $message; $timeout //= 5; $delay //= 0.1; # Wait for the device size to update my $time = 0; while ( $time < $timeout ) { if ( &$success() ) { if ( $DEBUG && $time > 0 ) { print $debug if $DEBUG >= 2; print ": done in $time sec\n"; } return 1; } if ( $DEBUG && $time == 0 ) { print $debug; print "\n" if $DEBUG >= 2; } select( undef, undef, undef, $delay ); $time += $delay; } print $debug if $DEBUG >= 2; print ": timeout after $time sec\n" if $DEBUG; die "Error :: Timeout while waiting for $message\n"; } sub prepare_api_params { my ( $parms ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::prepare_api_params\n" if $DEBUG >= 2; return $parms unless ref( $parms ) eq 'HASH'; my @temp; my $ref; my @ands; my $or; while ( my ( $key, $value ) = each( %$parms ) ) { $ref = ref $value; if ( $ref eq 'HASH' ) { @temp = (); while ( my ( $fname, $fvalue ) = each( %$value ) ) { $ref = ref $fvalue; if ( $ref eq '' ) { $fvalue = [ split( ',', $fvalue ) ]; } else { die "Error :: Unsupported condition type: $ref" if $ref ne 'ARRAY'; } $or = $#$fvalue > 0; $fvalue = join( ' or ', map { "$fname='$_'" } @$fvalue ); $fvalue = '(' . $fvalue . ')' if $or; push @temp, $fvalue; } $value = join( ' and ', @temp ); } else { $value = join( ',', @$value ) if $ref eq 'ARRAY'; } push @ands, uri_escape( $key ) . '=' . uri_escape( $value ); } return join( '&', @ands ); } sub purestorage_name_prefix { my ( $scfg ) = @_; my $ckey = '_vnprefix'; my $prefix = $scfg->{ $ckey }; if ( !defined( $prefix ) ) { my %parms = ( vgname => '/', podname => '::' ); my $value; my $pkey = ''; while ( my ( $key, $suffix ) = each( %parms ) ) { $value = $scfg->{ $key }; if ( defined( $value ) ) { die "Error :: Cannot have both \"$pkey\" and \"$key\" provided at the same time\n" if $pkey ne ''; die "Error :: Invalid \"$key\" parameter value \"$value\"\n" if $value !~ m/^\w([\w-]*\w)?$/; $prefix = $value . $suffix; $pkey = $key; } } $prefix = '' if $pkey eq ''; # allow no prefix $pkey = 'vnprefix'; $value = $scfg->{ $pkey }; if ( defined( $value ) ) { $prefix .= $value; die "Error :: Invalid \"$pkey\" parameter value \"$value\"\n" if $prefix !~ m/^\w([\w-]*\w)?((\/|::)(\w[\w-]*)?)?$/; } $scfg->{ $ckey } = $prefix; } return $prefix; } sub purestorage_name { my ( $scfg, $volname, $snapname ) = @_; my $name = length( $volname ) ? purestorage_name_prefix( $scfg ) . $volname : ''; if ( length( $snapname ) ) { my $snap = $snapname; $snap =~ s/^(veeam_)/veeam-/; # s/_/-/g; $snap = 'snap-' . $snap unless defined $1; $name .= '.' if $name ne ''; $name .= $snap; } print 'Debug :: purestorage_name ::', ( defined( $volname ) ? ' name="' . $volname . '"' : '' ), ( defined( $snapname ) ? ' snap="' . $snapname . '"' : '' ), ' => "' . $name . '"', "\n" if $DEBUG >= 2; return $name; } sub get_device_path_wwn { my ( $serial ) = @_; die 'Error :: Volume serial is missing' unless length( $serial ); # Construct the WWN path my $wwn = lc( $purestorage_wwn_prefix . $serial ); my $path = '/dev/disk/by-id/wwn-0x' . substr( $wwn, -32 ); return ( $path, $wwn ); } sub get_device_size { my ( $device ) = @_; print "Debug :: get_device_size($device)\n" if $DEBUG; my $path = '/sys/block/' . basename( $device ) . '/size'; my $size = file_read_firstline( $path ) << 9; print "Debug :: Device \"$device\" size is $size bytes\n" if $DEBUG; return $size; } sub device_op { my ( $device_path, $op, $value ) = @_; open( my $fh, '>', $device_path . '/' . $op ) or die "Error :: Could not open file \"$device_path/$op\" for writing.\n"; print $fh $value; close( $fh ); } sub block_device_action { my ( $action, @devices ) = @_; print "Debug :: block_device_action($action,@devices)\n" if $DEBUG; foreach my $device ( @devices ) { if ( $device !~ /^(sd[a-z]+)$/ ) { warn "Warning :: Unexpected device name in block_device_action() => $action $device)\n"; next; } $device = $1; # untaint my $device_path = '/sys/block/' . $device . '/device'; if ( $action eq 'remove' ) { print "Debug :: Removing device: $device\n" if $DEBUG; exec_command( [ 'blockdev', '--flushbufs', '/dev/' . $device ] ); device_op( $device_path, 'state', 'offline' ); device_op( $device_path, 'delete', '1' ); } elsif ( $action eq 'rescan' ) { print "Debug :: Rescanning: $device\n" if $DEBUG; device_op( $device_path, 'rescan', '1' ); } else { die "Error :: Unsuported acitonin block_device_action() => $action\n"; } } } sub block_device_slaves { my ( $path ) = @_; my $device_path = abs_path( $path ); die "Error :: Can't resolve device path for $path\n" unless $device_path =~ /^([\/a-zA-Z0-9_\-\.]+)$/; $device_path = $1; # untaint print "Debug :: Device path resolved to \"$device_path\".\n" if $DEBUG; my $device_name = basename( $device_path ); my $slaves_path = '/sys/block/' . $device_name . '/slaves'; my @slaves; if ( -d $slaves_path ) { opendir( my $dh, $slaves_path ) or die "Cannot open directory: $!"; @slaves = grep { !/^\.\.?$/ } readdir( $dh ); closedir( $dh ); } if ( @slaves ) { print "Debug :: Disk \"$device_name\" slaves: " . join( ', ', @slaves ) . "\n" if $DEBUG; } else { warn "Warning :: Disk \"$device_name\" has no slaves.\n"; push @slaves, $device_name; } return $device_path, @slaves; } sub cleanup_lvm_on_device { my ( $wwid ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::cleanup_lvm_on_device\n" if $DEBUG; my $cleaned = 0; my @dm_devices; eval { run_command( [ get_command_path( 'dmsetup' ), 'ls' ], outfunc => sub { my $line = shift; if ( $line =~ /^(\S+)\s+\(/ ) { push @dm_devices, $1; } } ); }; return 0 if $@; my @lvm_to_remove; foreach my $dm ( @dm_devices ) { next if $dm =~ /^${wwid}(-part\d+)?$/; my $deps = ''; eval { run_command( [ get_command_path( 'dmsetup' ), 'deps', '-o', 'devname', $dm ], outfunc => sub { $deps .= shift; }, errfunc => sub { } ); }; if ( $deps =~ /${wwid}/ ) { push @lvm_to_remove, $dm; } } foreach my $lvm ( reverse sort @lvm_to_remove ) { print "Debug :: Removing LVM device: $lvm\n" if $DEBUG; my $removed = 0; eval { run_command( [ get_command_path( 'dmsetup' ), 'remove', $lvm ], errfunc => sub { } ); $removed = 1; }; if ( !$removed ) { eval { run_command( [ get_command_path( 'dmsetup' ), 'remove', '--force', $lvm ], errfunc => sub { } ); $removed = 1; }; } $cleaned++ if $removed; warn "Warning :: Failed to remove LVM device $lvm\n" unless $removed; } return $cleaned; } sub cleanup_partitions_on_device { my ( $wwid ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::cleanup_partitions_on_device\n" if $DEBUG; my $cleaned = 0; my $dm_path = '/dev/mapper/' . $wwid; eval { run_command( [ get_command_path( 'kpartx' ), '-d', $dm_path ], errfunc => sub { } ); $cleaned++; }; opendir( my $dh, '/dev/mapper' ) or return $cleaned; my @partitions = grep { /^${wwid}-part\d+$/ } readdir( $dh ); closedir( $dh ); foreach my $part ( reverse sort @partitions ) { print "Debug :: Removing partition: $part\n" if $DEBUG; eval { run_command( [ get_command_path( 'dmsetup' ), 'remove', '--force', $part ], errfunc => sub { } ); $cleaned++; }; warn "Warning :: Failed to remove partition $part\n" if $@; } return $cleaned; } ### BLOCK: Token cache management => PVE::Storage::Custom::PureStoragePlugin::sub::token_cache # # Race condition mitigation strategy: # 1. Jitter in is_token_valid() spreads refresh timing across nodes (±2.5%) # 2. Read-check-write pattern in save_token_to_cache() prevents overwriting newer tokens # 3. On 401 error, re-check file cache before requesting new token (another node may have refreshed) # 4. HTTP retry on 401 with max_retries=1 prevents infinite loops # 5. pmxcfs replication is eventually consistent (typically <1s) # # Scenario: Two nodes refresh simultaneously # - Node A and Node B both see expired token at ~80% TTL (with jitter spread) # - Node A gets token T1, Node B gets token T2 # - Node B tries to write T2, but sees T1 is already cached (created <5s ago), skips write # - Both nodes use T1 from cache → success # # Edge case: If somehow both write (race in pmxcfs replication) # - Node A has T1 in memory, but file has T2 # - Node A gets 401 on next request # - Node A re-reads cache, finds valid T2, uses it # - Success without extra API call # # Error scenarios: # 1. Node B login fails (network/API error) during simultaneous refresh # - Node A successfully cached token T1 # - Node B checks cache after login error, finds T1 # - Node B uses T1 → continues operating # - No service disruption # # 2. Worst case: Cache file deleted/corrupted during race # - Node gets 401, cache re-read fails # - Node requests new token (1 extra API call) # - Result: At most 1 extra API call, system remains operational sub get_token_cache_path { my ( $storeid, $array_index ) = @_; my $cache_dir = '/etc/pve/priv/purestorage'; # Create cache directory if it doesn't exist if ( !-d $cache_dir ) { eval { File::Path::make_path( $cache_dir, { mode => 0700 } ); print "Debug :: Created token cache directory: $cache_dir\n" if $DEBUG; }; if ( $@ ) { warn "Warning :: Failed to create token cache directory $cache_dir: $@\n"; return undef; } } return "$cache_dir/${storeid}_array${array_index}.json"; } sub read_token_cache { my ( $cache_path ) = @_; return undef unless defined $cache_path; if ( !-f $cache_path ) { print "Debug :: Token cache file does not exist: $cache_path\n" if $DEBUG >= 2; return undef; } my $token_data; eval { my $json_text = PVE::Tools::file_get_contents( $cache_path ); $token_data = decode_json( $json_text ); print "Debug :: Read token cache from: $cache_path\n" if $DEBUG >= 1; }; if ( $@ ) { warn "Warning :: Failed to read token cache from $cache_path: $@\n"; # Delete corrupt cache file eval { unlink $cache_path }; return undef; } return $token_data; } sub write_token_cache { my ( $cache_path, $token_data ) = @_; return unless defined $cache_path; my $json_text = encode_json( $token_data ); # Atomic write: write to temp file, then rename my $temp_path = "$cache_path.tmp.$$"; eval { my $fh = IO::File->new( $temp_path, 'w', 0600 ) or die "Cannot create temp file $temp_path: $!\n"; print $fh $json_text . "\n"; $fh->close(); rename( $temp_path, $cache_path ) or die "Cannot rename $temp_path to $cache_path: $!\n"; print "Debug :: Wrote token cache to: $cache_path\n" if $DEBUG >= 1; }; if ( $@ ) { warn "Warning :: Failed to write token cache to $cache_path: $@\n"; # Clean up temp file if it exists eval { unlink $temp_path if -f $temp_path }; die $@; } } sub is_token_valid { my ( $token_data, $ttl ) = @_; return 0 unless defined $token_data; return 0 unless defined $token_data->{ auth_token }; return 0 unless defined $token_data->{ created_at }; return 0 unless defined $token_data->{ ttl }; my $now = time(); my $age = $now - $token_data->{ created_at }; # Add jitter (±5%) to refresh threshold to prevent thundering herd # when multiple nodes check token expiration simultaneously my $jitter = 0.05 * ( rand() - 0.5 ); # -2.5% to +2.5% my $refresh_threshold = $ttl * ( 0.8 + $jitter ); print "Debug :: Token validation: now=$now, created_at=$token_data->{ created_at }, age=${age}s, threshold=${refresh_threshold}s\n" if $DEBUG >= 2; if ( $age < $refresh_threshold ) { print "Debug :: Token is valid (age: ${age}s)\n" if $DEBUG >= 1; return 1; } print "Debug :: Token needs refresh (age: ${age}s >= threshold: ${refresh_threshold}s)\n" if $DEBUG >= 1; return 0; } sub cleanup_expired_cache { my ( $cache_path, $ttl ) = @_; return unless defined $cache_path; return unless -f $cache_path; my $token_data = read_token_cache( $cache_path ); return unless defined $token_data; if ( defined $token_data->{ expires_at } ) { my $now = time(); if ( $now >= $token_data->{ expires_at } ) { print "Debug :: Cleaning up expired token cache: $cache_path\n" if $DEBUG; eval { unlink $cache_path }; if ( $@ ) { warn "Warning :: Failed to delete expired cache $cache_path: $@\n"; } } } } ### BLOCK: API Helper functions => PVE::Storage::Custom::PureStoragePlugin::sub::api_helpers sub load_auth_token { my ( $storeid, $array_index, $scfg ) = @_; my $cache_path = defined( $storeid ) ? get_token_cache_path( $storeid, $array_index ) : undef; my $ttl = $scfg->{ token_ttl } || 3600; # Try in-memory cache first (fastest, no I/O) my $mem_token_key = '_auth_token' . $array_index; my $mem_request_id_key = '_request_id' . $array_index; if ( defined( $scfg->{ $mem_token_key } ) && $scfg->{ $mem_token_key } ne '' ) { print "Debug :: Using cached token from memory\n" if $DEBUG >= 2; return ( $scfg->{ $mem_token_key }, $scfg->{ $mem_request_id_key }, $cache_path, $ttl ); } # Try file cache if ( $cache_path ) { my $cached_token = read_token_cache( $cache_path ); if ( $cached_token && is_token_valid( $cached_token, $ttl ) ) { my $age = time() - $cached_token->{ created_at }; print "Debug :: Using cached token from file (age: ${age}s)\n" if $DEBUG >= 1; # Update in-memory cache for faster access next time $scfg->{ $mem_token_key } = $cached_token->{ auth_token }; $scfg->{ $mem_request_id_key } = $cached_token->{ request_id }; return ( $cached_token->{ auth_token }, $cached_token->{ request_id }, $cache_path, $ttl ); } } # File cache is expired or missing, return undef to force new token request return ( undef, undef, $cache_path, $ttl ); } sub save_token_to_cache { my ( $config, $token_state ) = @_; # Only save if this was a login request return unless $token_state == TOKEN_STATE_LOGIN; # Only save if we have cache path and token return unless $config->{ cache_path } && $config->{ auth_token }; my $now = time(); my $ttl = $config->{ ttl } || 3600; my $token_data = { auth_token => $config->{ auth_token }, request_id => $config->{ request_id }, created_at => $now, ttl => $ttl, expires_at => $now + $ttl }; eval { # Race condition mitigation: check if another node already wrote a newer token my $existing = read_token_cache( $config->{ cache_path } ); if ( $existing && $existing->{ created_at } > $token_data->{ created_at } - 5 ) { # Another node wrote a token within last 5 seconds, use that instead print "Debug :: Another node already cached a token, skipping write\n" if $DEBUG >= 2; return; } write_token_cache( $config->{ cache_path }, $token_data ); print "Debug :: Token cached to file: $config->{ cache_path }\n" if $DEBUG >= 1; }; if ( $@ ) { warn "Warning :: Failed to write token cache: $@\n"; } } sub cleanup_token_cache { my ( $config ) = @_; return unless $config->{ cache_path }; eval { cleanup_expired_cache( $config->{ cache_path }, $config->{ ttl } || 3600 ); }; } sub is_ignorable_error { my ( $action, $content ) = @_; my $ignore = $action->{ ignore }; return 0 unless defined $ignore; # Normalize to array $ignore = [$ignore] unless ref( $ignore ) eq 'ARRAY'; # Check if error message is in ignore list my $error_msg = $content->{ errors }->[0]->{ message } // ''; return grep { $_ eq $error_msg } @$ignore; } sub try_cached_token { my ( $config ) = @_; return 0 unless $config->{ cache_path }; my $cached_token = read_token_cache( $config->{ cache_path } ); return 0 unless $cached_token && $cached_token->{ auth_token }; return 0 unless is_token_valid( $cached_token, $config->{ ttl } || 3600 ); my $age = time() - $cached_token->{ created_at }; print "Debug :: Using cached token from file (age: ${age}s)\n" if $DEBUG >= 1; $config->{ auth_token } = $cached_token->{ auth_token }; $config->{ request_id } = $cached_token->{ request_id }; return 1; } ### BLOCK: Local multipath => PVE::Storage::Custom::PureStoragePlugin::sub::s sub purestorage_api_call { my ( $scfg, $action, $all, $storeid ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_api_call\n" if $DEBUG >= 3; $all //= 0; my $ua = LWP::UserAgent->new( timeout => 15 ); $ua->ssl_opts( verify_hostname => 0, SSL_verify_mode => 0x00 ) unless $scfg->{ check_ssl }; my $type = $action->{ type }; my $login = $type eq 'login' ? 1 : 0; my $params = prepare_api_params( $action->{ params } ); my $path = $type; $path .= '?' . $params if length( $params ); my $method = $action->{ method }; my $body = $action->{ body }; my $error; my $content; my $url; my @urls = split( ',', $scfg->{ address } // '' ); my @tokens = split( ',', $scfg->{ token } // '' ); my $array_count = 0; my $success_count = 0; my $last_success_error = ERROR_SUCCESS; my $last_success_content; foreach my $i ( 0, 1 ) { $url = $urls[$i] // ''; my $token = $tokens[$i] // ''; next if $i && $url eq '' && $token eq ''; $array_count++; my $cf = $url eq '' ? 'address' : $token eq '' ? 'token' : ''; die "Error :: Pure Storage \"$cf\" parameter" . ( $i == 0 ? '' : ' for second array' ) . " is not defined.\n" unless $cf eq ''; # Load auth token (file cache → memory cache → undef) my ( $auth_token, $request_id, $cache_path, $ttl ) = load_auth_token( $storeid, $i, $scfg ); my $config = { ua => $ua, url => $url, token => $token, auth_token => $auth_token, request_id => $request_id, cache_path => $cache_path, ttl => $ttl }; ( $error, $content ) = purestorage_http_request( $config, $path, $method, $login, $body ); # Handle token update if ( $error == ERROR_TOKEN_UPDATED ) { $scfg->{ '_auth_token' . $i } = $config->{ auth_token }; $scfg->{ '_request_id' . $i } = $config->{ request_id }; } # Handle ignorable API errors elsif ( $error == ERROR_API_ERROR && is_ignorable_error( $action, $content ) ) { $error = ERROR_SUCCESS; } # Track success for this array if ( $error <= ERROR_SUCCESS ) { $success_count++; $last_success_error = $error; $last_success_content = $content; print "Debug :: Array " . ( $i + 1 ) . " ($url) succeeded\n" if $DEBUG >= 2; } # Stop on critical authentication error (cannot continue) if ( $error == ERROR_AUTH_FAILED ) { last; } # Handle API errors in Active Cluster mode if ( $error == ERROR_API_ERROR ) { if ( $all && $success_count > 0 ) { # Continue to next array - partial success acceptable in Active Cluster print "Warning :: Array " . ( $i + 1 ) . " ($url) failed but array(s) succeeded. Continuing...\n"; next; } else { last; } } # Stop on success if not processing all arrays last if $error <= ERROR_SUCCESS && !$all; } # Use last successful response if we processed multiple arrays if ( $all && $success_count > 0 ) { $error = $last_success_error; $content = $last_success_content; print "Debug :: Processed $array_count array(s), $success_count succeeded\n" if $DEBUG >= 2; } # Handle fatal errors # For operations on all arrays (Active Cluster), fail only if all arrays failed if ( $error > ERROR_SUCCESS ) { if ( $all && $success_count > 0 ) { # At least one array succeeded, so operation is partially successful # This is acceptable for Active Cluster scenarios print "Warning :: Operation completed on $success_count of $array_count array(s). Some arrays may have failed.\n"; return $last_success_content; } my $message = $error == ERROR_AUTH_FAILED ? 'Authentication' : $action->{ name } || "Action '$type' (method '$method')"; $message = substr( $message, 0, 1 ) eq uc( substr( $message, 0, 1 ) ) ? $message . ' failed' : 'Failed to ' . $message; $message = 'PureStorage API :: ' . $message if $error == ERROR_API_ERROR; die "Error :: $message.\n" . "=> Trace:\n" . "==> address: " . $url . "\n" . ( $content ? "==> Message: " . Dumper( $content ) : '' ); } return $content; } sub purestorage_http_request { my ( $config, $path, $method, $login, $body ) = @_; my $headers = HTTP::Headers->new( 'Content-Type' => 'application/json' ); # Determine token state my $token_state; if ( $login ) { $token_state = TOKEN_STATE_LOGIN; $headers->header( 'api-token' => $config->{ token } ); } elsif ( $config->{ auth_token } ) { $token_state = TOKEN_STATE_CACHED; } else { $token_state = TOKEN_STATE_NEEDED; } my $error; my $response; my $content; my $retry_count = 0; my $max_retries = 1; # Allow one retry for token refresh # Retry loop for token expiration (max 1 retry) while ( $retry_count <= $max_retries ) { # Obtain token if needed if ( $token_state > TOKEN_STATE_LOGIN ) { if ( $token_state == TOKEN_STATE_NEEDED ) { # Check cache first (race condition mitigation) unless ( try_cached_token( $config ) ) { # Request new token print "Debug :: Requesting new session token\n" if $DEBUG >= 1; ( $error, $content ) = purestorage_http_request( $config, 'login', 'POST', 1 ); # On failure, try cache again (another node may have succeeded) if ( $error > ERROR_SUCCESS ) { print "Debug :: Login failed, checking if another node cached a token\n" if $DEBUG >= 2; unless ( try_cached_token( $config ) ) { return ( $error, $content ); } } } $token_state = TOKEN_STATE_CACHED; } else { print "Debug :: Using existing session token\n" if $DEBUG >= 2; } $headers->header( 'x-auth-token' => $config->{ auth_token } ); } $headers->header( 'X-Request-ID' => $config->{ request_id } ) if $config->{ request_id }; # Execute HTTP request my $request = HTTP::Request->new( $method, $config->{ url } . '/api/' . $PSFA_API . '/' . $path, $headers, length( $body ) ? encode_json( $body ) : undef ); $response = $config->{ ua }->request( $request ); # Handle 401 Unauthorized (token expired) $error = $response->is_success ? ERROR_SUCCESS : ERROR_API_ERROR; if ( $error && $token_state == TOKEN_STATE_CACHED && $response->code == 401 ) { $retry_count++; if ( $retry_count <= $max_retries ) { print "Debug :: Session token expired (401), retry $retry_count/$max_retries\n" if $DEBUG >= 1; # Save current token to detect if cache has newer version my $old_token = $config->{ auth_token }; # Try cache first - another node may have already refreshed if ( try_cached_token( $config ) && $config->{ auth_token } ne $old_token ) { print "Debug :: Using refreshed token from another node\n" if $DEBUG >= 2; $token_state = TOKEN_STATE_CACHED; next; } # No fresh token in cache, request new one cleanup_token_cache( $config ); $token_state = TOKEN_STATE_NEEDED; next; } else { print "Debug :: Max retries ($max_retries) reached, giving up\n" if $DEBUG >= 1; last; } } last; } # Process successful response $headers = $response->headers; if ( $error == ERROR_SUCCESS ) { if ( $token_state == TOKEN_STATE_LOGIN ) { # Extract tokens from login response $config->{ auth_token } = $headers->header( 'x-auth-token' ) or die "Error :: PureStorage API :: Header 'x-auth-token' is missing.\n"; $config->{ request_id } = $headers->header( 'x-request-id' ); # Save token to cache save_token_to_cache( $config, $token_state ); } # Signal that token was updated $error = ERROR_TOKEN_UPDATED if $token_state < TOKEN_STATE_CACHED; } # Parse response content $content = $response->decoded_content; my $content_type = $headers->header( 'Content-Type' ) // ''; if ( $content_type =~ /application\/json/ ) { $content = decode_json( $content ); } else { # Non-JSON response indicates connectivity/network error $error = $login ? ERROR_AUTH_FAILED : ERROR_NETWORK_ERROR if $error == ERROR_API_ERROR; $content = { response => $content }; } return ( $error, $content ); } sub purestorage_list_volumes { my ( $class, $scfg, $vmid, $storeid, $destroyed ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_list_volumes\n" if $DEBUG >= 2; $vmid = '*' unless defined( $vmid ); my $names = "vm-$vmid-disk-*,vm-$vmid-cloudinit,vm-$vmid-state-*"; return $class->purestorage_get_volumes( $scfg, $names, $storeid, $destroyed ); } sub purestorage_get_volumes { my ( $class, $scfg, $names, $storeid, $destroyed ) = @_; my $filter = { name => [ map { purestorage_name( $scfg, $_ ) } split( ',', $names ) ] }; $filter->{ destroyed } = $destroyed ? 'true' : 'false' if defined $destroyed; my $action = { name => $names =~ m/[*,]/ ? 'list volumes' : 'get volume information', type => 'volumes', method => 'GET', params => { filter => $filter } }; my $response = purestorage_api_call( $scfg, $action, 0, $storeid ); my $pref_len = length( purestorage_name_prefix( $scfg ) ); my @volumes = map { my $volname = substr( $_->{ name }, $pref_len ); my ( undef, undef, $volvm ) = $class->parse_volname( $volname ); my $ctime = int( $_->{ created } / 1000 ); { name => $volname, vmid => $volvm, serial => $_->{ serial }, size => $_->{ provisioned } || 0, used => $_->{ space }->{ total_used } || 0, ctime => $ctime, volid => $storeid ? "$storeid:$volname" : $volname, format => 'raw' } } @{ $response->{ items } }; return \@volumes; } sub purestorage_get_volume_info { my ( $class, $scfg, $volname, $storeid, $destroyed ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_get_volume_info\n" if $DEBUG; my $volumes = $class->purestorage_get_volumes( $scfg, $volname, $storeid, $destroyed ); foreach my $volume ( @$volumes ) { return $volume; } return undef; } sub purestorage_get_existing_volume_info { my ( $class, $scfg, $volname, $storeid ) = @_; return $class->purestorage_get_volume_info( $scfg, $volname, $storeid, 0 ); } sub purestorage_get_wwn { my ( $class, $scfg, $volname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_get_wwn\n" if $DEBUG; my $volume = $class->purestorage_get_existing_volume_info( $scfg, $volname ); return get_device_path_wwn( $volume->{ serial } ) if $volume; warn "Warning :: Can't get volume \"$volname\" info\n"; return ( '', '' ); } sub purestorage_volume_connection { my ( $class, $storeid, $scfg, $volname, $mode ) = @_; my $method = $mode ? 'POST' : 'DELETE'; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_volume_connection :: $method\n" if $DEBUG; my $hname = PVE::INotify::nodename(); my $hgsuffix = $scfg->{ hgsuffix } // $default_hgsuffix; $hname .= "-" . $hgsuffix if $hgsuffix ne ""; my $name; my $ignore; if ( $mode ) { $name = 'create volume connection'; $ignore = 'Connection already exists.'; } else { $name = 'delete volume connection'; $ignore = [ 'Volume has been destroyed.', 'Connection does not exist.' ]; } my $action = { name => $name, type => 'connections', method => $method, ignore => $ignore, params => { host_names => $hname, volume_names => purestorage_name( $scfg, $volname ) } }; # For Active Cluster: connect/disconnect on all arrays (both primary and secondary) # This ensures volumes are accessible from both arrays in Active Cluster configuration my $response = purestorage_api_call( $scfg, $action, 1, $storeid ); my $message = ( $response->{ errors } ? 'already ' : '' ) . ( $mode ? 'connected to' : 'disconnected from' ); print "Info :: Volume \"$volname\" is $message host \"$hname\" on all arrays.\n"; return 1; } sub purestorage_create_volume { my ( $class, $scfg, $volname, $size, $storeid ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_create_volume\n" if $DEBUG; my $action = { name => 'create volume', type => 'volumes', method => 'POST', params => { names => purestorage_name( $scfg, $volname ) }, body => { provisioned => $size } }; my $response = purestorage_api_call( $scfg, $action, 0, $storeid ); my $serial = $response->{ items }->[0]->{ serial } or die "Error :: Failed to retrieve volume serial"; print "Info :: Volume \"$volname\" is created (serial=$serial).\n"; return 1; } sub purestorage_remove_volume { my ( $class, $scfg, $volname, $storeid, $eradicate ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_remove_volume\n" if $DEBUG; if ( $volname =~ /^vm-(\d+)-(cloudinit|state-.+)/ ) { $eradicate = 1; } else { $eradicate //= 0; } # Clean up local device mappings before removing from Pure Storage my $volume = $class->purestorage_get_volume_info( $scfg, $volname, $storeid, 0 ); if ( $volume && $volume->{ serial } ) { my ( $path, $wwid ) = get_device_path_wwn( $volume->{ serial } ); if ( $wwid ne '' && -e "/dev/mapper/$wwid" ) { print "Debug :: Cleaning up local device mappings for $wwid\n" if $DEBUG; # 1. Remove LVM mappings on top of the device cleanup_lvm_on_device( $wwid ); # 2. Remove partition mappings cleanup_partitions_on_device( $wwid ); # 3. Remove multipath device if ( multipath_check( $wwid ) ) { print "Debug :: Removing multipath device $wwid\n" if $DEBUG; exec_command( [ 'multipath', '-f', $wwid ], 0 ); } } } # Disconnect volume from all hosts on all arrays before destroying # For Active Cluster: get connections from first array (they are synced) and disconnect from all hosts print "Debug :: Disconnecting volume from all hosts on all arrays\n" if $DEBUG >= 2; my $pure_volname = purestorage_name( $scfg, $volname ); # Get list of all connections (from first array - in Active Cluster connections are synced) my $connections_action = { name => 'list volume connections', type => 'connections', method => 'GET', params => { volume_names => $pure_volname } }; my $connections_response = purestorage_api_call( $scfg, $connections_action, 0, $storeid ); my @connections = @{ $connections_response->{ items } || [] }; if ( @connections ) { print "Debug :: Found " . scalar( @connections ) . " connection(s) for volume \"$volname\"\n" if $DEBUG >= 1; # Collect unique hostnames my %unique_hosts; foreach my $conn ( @connections ) { my $hostname = $conn->{ host }->{ name }; $unique_hosts{ $hostname } = 1; } # Disconnect from each unique host on all arrays foreach my $hostname ( keys %unique_hosts ) { print "Debug :: Disconnecting from host \"$hostname\" on all arrays\n" if $DEBUG >= 2; my $disconnect_action = { name => 'delete volume connection', type => 'connections', method => 'DELETE', ignore => [ 'Volume has been destroyed.', 'Connection does not exist.' ], params => { host_names => $hostname, volume_names => $pure_volname } }; # For Active Cluster: disconnect from this host on all arrays purestorage_api_call( $scfg, $disconnect_action, 1, $storeid ); } print "Info :: Volume \"$volname\" disconnected from " . scalar( keys %unique_hosts ) . " host(s) on all arrays.\n"; } else { print "Debug :: No connections found for volume \"$volname\"\n" if $DEBUG >= 2; } my $params = { names => $pure_volname }; my $action = { name => 'destroy volume', type => 'volumes', method => 'PATCH', ignore => 'Volume has been deleted.', params => $params, body => { destroyed => \1 } }; # For Active Cluster: destroy volume on all arrays my $response = purestorage_api_call( $scfg, $action, 1, $storeid ); my $message = ( $response->{ errors } ? 'already ' : '' ) . 'destroyed'; print "Info :: Volume \"$volname\" is $message.\n"; if ( $eradicate ) { $action = { name => 'eradicate volume', type => 'volumes', method => 'DELETE', ignore => 'Eradication is disabled.', params => $params, }; # For Active Cluster: eradicate volume on all arrays purestorage_api_call( $scfg, $action, 1, $storeid ); print "Info :: Volume \"$volname\" is eradicated.\n"; } return 1; } sub purestorage_resize_volume { my ( $class, $scfg, $storeid, $volname, $size ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_resize_volume\n" if $DEBUG; my $action = { name => 'resize volume', type => 'volumes', method => 'PATCH', params => { names => purestorage_name( $scfg, $volname ) }, body => { provisioned => $size } }; my $response = purestorage_api_call( $scfg, $action, 0, $storeid ); my $serial = $response->{ items }->[0]->{ serial } or die "Error :: Failed to retrieve volume serial"; my ( $path, $wwid ) = get_device_path_wwn( $serial ); # return early if the volume is not mapped (normally should not happen) return $size unless $path ne '' && -b $path; my ( $device_path, @slaves ) = block_device_slaves( $path ); # Iterate through slaves and rescan each device block_device_action( 'rescan', @slaves ); if ( multipath_check( $wwid ) ) { print "Debug :: Device \"$wwid\" is a multipath device. Proceeding with resizing.\n" if $DEBUG; exec_command( [ 'multipathd', 'resize', 'map', $wwid ] ); } print "Debug :: Expected size = $size\n" if $DEBUG; my $new_size; my $updated_size = sub { $new_size = get_device_size( $device_path ); return $new_size >= $size; }; # FIXME: With the current implementation we may not need to wait wait_for( $updated_size, "volume \"$volname\" size update" ); print "Debug :: New size detected for volume \"$volname\": $new_size bytes.\n" if $DEBUG; print "Info :: Volume \"$volname\" is resized.\n"; return $new_size; } sub purestorage_rename_volume { my ( $class, $scfg, $storeid, $source_volname, $target_volname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_rename_volume\n" if $DEBUG; my $action = { name => 'rename volume', type => 'volumes', method => 'PATCH', params => { names => purestorage_name( $scfg, $source_volname ) }, body => { name => purestorage_name( $scfg, $target_volname ) } }; purestorage_api_call( $scfg, $action, 0, $storeid ); print "Info :: Volume \"$source_volname\" is renamed to \"$target_volname\".\n"; return 1; } sub purestorage_snap_volume_create { my ( $class, $scfg, $storeid, $snap_name, $volname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_snap_volume_create\n" if $DEBUG; my $action = { name => 'create volume snapshot', type => 'volume-snapshots', method => 'POST', params => { source_names => purestorage_name( $scfg, $volname ), suffix => purestorage_name( $scfg, undef, $snap_name ) } }; purestorage_api_call( $scfg, $action, 0, $storeid ); print "Info :: Volume \"$volname\" snapshot \"$snap_name\" is created.\n"; return 1; } sub purestorage_volume_restore { my ( $class, $scfg, $storeid, $volname, $svolname, $snap, $overwrite ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_volume_restore\n" if $DEBUG; my $params = { names => purestorage_name( $scfg, $volname ) }; $params->{ overwrite } = $overwrite ? 'true' : 'false' if defined $overwrite; my $action = { name => 'restore volume', type => 'volumes', method => 'POST', params => $params, body => { source => { name => purestorage_name( $scfg, $svolname, $snap ) } } }; purestorage_api_call( $scfg, $action, 0, $storeid ); my $source = length( $snap ) ? 'snapshot "' . $snap . '"' : ''; if ( $volname ne $svolname ) { $source .= ' of ' if $source ne ''; $source .= 'volume "' . $svolname . '"'; } $source = ' from ' . $source if $source ne ''; print "Info :: Volume \"$volname\" is restored$source.\n"; } sub purestorage_snap_volume_delete { my ( $class, $scfg, $storeid, $snap_name, $volname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_snap_volume_delete\n" if $DEBUG; my $params = { names => purestorage_name( $scfg, $volname, $snap_name ) }; my $action = { name => 'destroy volume snapshot', type => 'volume-snapshots', method => 'PATCH', ignore => [ 'Volume snapshot has been destroyed. It can be recovered by purevol recover and eradicated by purevol eradicate.', 'No such volume or snapshot.' ], params => $params, body => { destroyed => \1 } }; my $response = purestorage_api_call( $scfg, $action, 0, $storeid ); my $message = ( $response->{ errors } ? 'already ' : '' ) . 'destroyed'; print "Info :: Volume \"$volname\" snapshot \"$snap_name\" is $message.\n"; #FIXME: Pure FA API states that replication_snapshot is query (not body) parameter $action = { name => 'eradicate volume snapshot', type => 'volume-snapshots', method => 'DELETE', ignore => [ 'No such volume or snapshot.', 'Eradication is disabled.' ], params => $params, body => { replication_snapshot => \1 } }; $response = purestorage_api_call( $scfg, $action, 0, $storeid ); $message = ( $response->{ errors } ? 'already ' : '' ) . 'eradicated'; print "Info :: Volume \"$volname\" snapshot \"$snap_name\" is $message.\n"; return 1; } ### BLOCK: Storage implementation sub parse_volname { my ( $class, $volname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::parse_volname\n" if $DEBUG >= 3; if ( $volname =~ m/^(vm|base)-(\d+)-(\S+)$/ ) { my $vtype = ( $1 eq "vm" ) ? "images" : "base"; # Determine volume type my $vmid = $2; # Extract VMID my $name = $3; # Remaining part of the volume name # ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) return ( $vtype, $name, $vmid, undef, undef, undef, 'raw' ); } die "Error :: Invalid volume name ($volname).\n"; return 0; } sub filesystem_path { my ( $class, $scfg, $volname, $snapname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::filesystem_path\n" if $DEBUG; die "Error :: filesystem_path: snapshot is not implemented ($snapname)\n" if defined( $snapname ); # do we even need this? my ( $vtype, undef, $vmid ) = $class->parse_volname( $volname ); my ( $path, $wwid ) = $class->purestorage_get_wwn( $scfg, $volname ); if ( !defined( $path ) || !defined( $vmid ) || !defined( $vtype ) ) { return wantarray ? ( "", "", "", "" ) : ""; } return wantarray ? ( $path, $vmid, $vtype, $wwid ) : $path; } sub create_base { my ( $class, $storeid, $scfg, $volname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::create_base\n" if $DEBUG; die "Error :: Creating base image is currently unimplemented.\n"; } sub clone_image { my ( $class, $scfg, $storeid, $volname, $vmid, $snap ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::clone_image\n" if $DEBUG; my $name = $class->find_free_diskname( $storeid, $scfg, $vmid ); $class->purestorage_volume_restore( $scfg, $storeid, $name, $volname, $snap ); return $name; } sub find_free_diskname { my ( $class, $storeid, $scfg, $vmid, $fmt, $add_fmt_suffix ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::find_free_diskname\n" if $DEBUG; my $volumes = $class->purestorage_list_volumes( $scfg, $vmid, $storeid ); my @disk_list = map { $_->{ name } } @$volumes; return PVE::Storage::Plugin::get_next_vm_diskname( \@disk_list, $storeid, $vmid, undef, $scfg ); } sub alloc_image { my ( $class, $storeid, $scfg, $vmid, $fmt, $name, $size ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::alloc_image\n" if $DEBUG; # Check for supported format (only 'raw' is allowed) die "Error :: Unsupported format ($fmt).\n" if $fmt ne 'raw'; # Validate the name format, should start with 'vm-$vmid-disk' if ( defined( $name ) ) { die "Error :: Illegal name \"$name\" - should be \"vm-$vmid-(disk-*|cloudinit|state-*)\".\n" if $name !~ m/^vm-$vmid-(disk-|cloudinit|state-)/; } else { $name = $class->find_free_diskname( $storeid, $scfg, $vmid ); } # Check size (must be between 1MB and 4PB) if ( $size < 1024 ) { print "Info :: Size is too small ($size kb), adjusting to 1024 kb\n"; $size = 1024; } # Convert size from KB to bytes my $sizeB = $size * 1024; # KB => B if ( !$class->purestorage_create_volume( $scfg, $name, $sizeB, $storeid ) ) { die "Error :: Failed to create volume \"$name\".\n"; } return $name; } sub free_image { my ( $class, $storeid, $scfg, $volname, $isBase ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::free_image\n" if $DEBUG; $class->deactivate_volume( $storeid, $scfg, $volname ); $class->purestorage_remove_volume( $scfg, $volname, $storeid ); return undef; } sub list_images { my ( $class, $storeid, $scfg, $vmid, $vollist, $cache ) = @_; set_debug_from_config( $scfg ); print "Debug :: list_images ($storeid, vmid=" . ( $vmid // 'all' ) . ")\n" if $DEBUG >= 1; return $class->purestorage_list_volumes( $scfg, $vmid, $storeid, 0 ); } sub status { my ( $class, $storeid, $scfg, $cache ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::status\n" if $DEBUG; my $total; my $used; # If using pod with quota, get pod-specific capacity if ( defined $scfg->{ podname } && $scfg->{ podname } ne '' ) { my $podname = $scfg->{ podname }; print "Debug :: Getting pod quota for pod: $podname\n" if $DEBUG >= 2; my $action = { name => 'get pod space', type => 'pods/space', method => 'GET', params => { names => $podname } }; my $response = purestorage_api_call( $scfg, $action, 0, $storeid ); my $pod = $response->{ items }->[0]; if ( $pod ) { # Use quota_limit if set and non-zero, otherwise fall back to array capacity # quota_limit = 0 or undef means unlimited (no quota) my $quota = $pod->{ quota_limit }; $total = ( defined( $quota ) && $quota > 0 ) ? $quota : $pod->{ capacity }; $used = $pod->{ space }->{ total_physical }; my $quota_str = defined( $quota ) ? ( $quota > 0 ? $quota : 'unlimited' ) : 'not set'; print "Debug :: Pod quota_limit: $quota_str\n" if $DEBUG >= 2; } else { die "Error :: Pod \"$podname\" not found\n"; } } else { # Get array-wide capacity my $response = purestorage_api_call( $scfg, { name => 'get array space', type => 'arrays/space', method => 'GET' }, 0, $storeid ); my $array = $response->{ items }->[0]; $total = $array->{ capacity }; # total_physical - physically used space on the array (after deduplication and compression) # total_used - logically used space (before deduplication and compression) $used = $array->{ space }->{ total_physical }; } # Calculate free space my $free = $total - $used; # Mark storage as active my $active = 1; # Return total, free, used space and the active status return ( $total, $free, $used, $active ); } sub activate_storage { my ( $class, $storeid, $scfg, $cache ) = @_; set_debug_from_config( $scfg ); print "Debug :: activate_storage ($storeid)\n" if $DEBUG >= 1; return 1; } sub deactivate_storage { my ( $class, $storeid, $scfg, $cache ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::deactivate_storage\n" if $DEBUG; return 1; } sub volume_size_info { my ( $class, $scfg, $storeid, $volname, $timeout ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::volume_size_info\n" if $DEBUG; my $volume = $class->purestorage_get_existing_volume_info( $scfg, $volname ); #TODO: Consider moving this inside of purestorage_get_existing_volume_info() die "Error :: PureStorage API :: No volume data found for \"$volname\".\n" unless $volume; print "Debug :: Provisioned: " . $volume->{ size } . ", Used: " . $volume->{ used } . "\n" if $DEBUG; return wantarray ? ( $volume->{ size }, 'raw', $volume->{ used }, undef ) : $volume->{ size }; } sub map_volume { my ( $class, $storeid, $scfg, $volname, $snapname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::map_volume\n" if $DEBUG; my ( $path, $wwid ) = $class->purestorage_get_wwn( $scfg, $volname ); print "Debug :: Mapping volume \"$volname\" with WWN: " . uc( $wwid ) . ".\n" if $DEBUG; my $protocol = $scfg->{ protocol } // $default_protocol; if ( $protocol eq 'iscsi' || $protocol eq 'fc' ) { scsi_scan_new( $protocol ); } elsif ( $protocol eq 'nvme' ) { die "Error :: Protocol: \"$protocol\" isn't implemented yet.\n"; } else { die "Error :: Protocol: \"$protocol\" isn't a valid protocol.\n"; } my $path_exists = sub { return -e $path; }; # Wait for the device to appear wait_for( $path_exists, "volume \"$volname\" to map", 30 ); # we might end up with operational disk but without multipathing, e.g. # if unmapping was interrupted ('remove map' was already done, but slaves were not removed) if ( !multipath_check( $wwid ) ) { print "Debug :: Adding multipath map for device \"$wwid\"\n" if $DEBUG; exec_command( [ 'multipathd', 'add', 'map', $wwid ] ); # Wait for multipath to be fully established my $multipath_ready = sub { return multipath_check( $wwid ); }; wait_for( $multipath_ready, "multipath map for volume \"$volname\" to be ready", 30 ); } return $path; } sub unmap_volume { my ( $class, $storeid, $scfg, $volname, $snapname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::unmap_volume\n" if $DEBUG; my ( $path, $wwid ) = $class->purestorage_get_wwn( $scfg, $volname ); return 0 unless $path ne '' && -b $path; my ( $device_path, @slaves ) = block_device_slaves( $path ); # Ensure all data is flushed to disk for write-back cache environments print "Debug :: Flushing filesystem and device buffers for $device_path\n" if $DEBUG >= 2; exec_command( ['sync'] ); exec_command( [ 'blockdev', '--flushbufs', $device_path ] ); # Wait for udev events to settle, ensuring all async operations complete eval { exec_command( [ 'udevadm', 'settle', '--timeout=10' ] ) }; # Final sync to guarantee write-back cache is flushed exec_command( ['sync'] ); if ( multipath_check( $wwid ) ) { print "Debug :: Device \"$wwid\" is a multipath device. Proceeding with multipath removal.\n" if $DEBUG; # remove the link exec_command( [ 'multipathd', 'remove', 'map', $wwid ] ); } else { print "Debug :: Device \"$wwid\" is not a multipath device. Skipping multipath removal.\n" if $DEBUG; } # Iterate through slaves and remove each device block_device_action( 'remove', @slaves ); print "Debug :: Device \"$wwid\" is removed.\n" if $DEBUG; return 1; } sub activate_volume { my ( $class, $storeid, $scfg, $volname, $snapname, $cache ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::activate_volume\n" if $DEBUG; $class->purestorage_volume_connection( $storeid, $scfg, $volname, 1 ); $class->map_volume( $storeid, $scfg, $volname, $snapname ); return 1; } sub deactivate_volume { my ( $class, $storeid, $scfg, $volname, $snapname, $cache ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::deactivate_volume\n" if $DEBUG; $class->unmap_volume( $storeid, $scfg, $volname, $snapname ); $class->purestorage_volume_connection( $storeid, $scfg, $volname, 0 ); print "Info :: Volume \"$volname\" is deactivated.\n"; return 1; } sub volume_resize { my ( $class, $scfg, $storeid, $volname, $size, $running ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::volume_resize\n" if $DEBUG; warn "Debug :: New Size: $size\n" if $DEBUG; return $class->purestorage_resize_volume( $scfg, $storeid, $volname, $size ); } sub rename_volume { my ( $class, $scfg, $storeid, $source_volname, $target_vmid, $target_volname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::rename_volume\n" if $DEBUG; die "Error :: not implemented in storage plugin \"$class\".\n" if $class->can( 'api' ) && $class->api() < 10; if ( length( $target_volname ) ) { # See RBDPlugin.pm (note, currently PVE does not supply $target_volname parameter) my $volume = $class->purestorage_get_volume_info( $scfg, $target_volname, $storeid ); die "target volume '$target_volname' already exists\n" if $volume; } else { $target_volname = $class->find_free_diskname( $storeid, $scfg, $target_vmid ); } # we need to unmap source volume (see RBDPlugin.pm) $class->unmap_volume( $storeid, $scfg, $source_volname ); $class->purestorage_rename_volume( $scfg, $storeid, $source_volname, $target_volname ); return "$storeid:$target_volname"; } sub volume_import { my ( $class, $scfg, $storeid, $fh, $volname, $format, $snapshot, $base_snapshot, $with_snapshots, $allow_rename ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::volume_import\n" if $DEBUG; die "=> PVE::Storage::Custom::PureStoragePlugin::sub::volume_import not implemented!"; return 1; } sub volume_snapshot { my ( $class, $scfg, $storeid, $volname, $snap ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::volume_snapshot\n" if $DEBUG; $class->purestorage_snap_volume_create( $scfg, $storeid, $snap, $volname ); return 1; } sub volume_snapshot_rollback { my ( $class, $scfg, $storeid, $volname, $snap ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::volume_snapshot_rollback\n" if $DEBUG; $class->purestorage_volume_restore( $scfg, $storeid, $volname, $volname, $snap, 1 ); return 1; } sub volume_snapshot_delete { my ( $class, $scfg, $storeid, $volname, $snap ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::volume_snapshot_delete\n" if $DEBUG; $class->purestorage_snap_volume_delete( $scfg, $storeid, $snap, $volname ); return 1; } sub volume_has_feature { my ( $class, $scfg, $feature, $storeid, $volname, $snapname, $running ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::volume_has_feature\n" if $DEBUG; my $features = { copy => { current => 1, snap => 1 }, # full clone is possible clone => { current => 1, snap => 1 }, # linked clone is possible snapshot => { current => 1 }, # taking a snapshot is possible # template => { current => 1 }, # conversion to base image is possible sparseinit => { current => 1 }, # thin provisioning is supported rename => { current => 1 }, # renaming volumes is possible }; my ( $vtype, $name, $vmid, $basename, $basevmid, $isBase ) = $class->parse_volname( $volname ); my $key; if ( $snapname ) { $key = "snap"; } else { $key = $isBase ? "base" : "current"; } return 1 if $features->{ $feature }->{ $key }; return undef; } 1;