From c8737e0e9028ce23120feff7255f9600b2d3a6b3 Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Sat, 28 Dec 2024 16:22:33 -0500 Subject: [PATCH 01/30] Make hgsuffix optional --- PureStoragePlugin.pm | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 3aa3013..d73ff7e 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -31,6 +31,7 @@ $Data::Dumper::Indent = 1; # Outputs everything in one line $Data::Dumper::Useqq = 1; # Uses quotes for strings my $purestorage_wwn_prefix = "624a9370"; +my $default_hgsuffix = "pve"; my $DEBUG = 0; @@ -128,7 +129,7 @@ sub properties { hgsuffix => { description => "Host group suffx.", type => "string", - default => "pve" + default => $default_hgsuffix }, address => { description => "PureStorage Management IP address or DNS name.", @@ -151,7 +152,7 @@ sub options { address => { fixed => 1 }, token => { fixed => 1 }, - hgsuffix => { fixed => 1 }, + hgsuffix => { optional => 1 }, vgname => { fixed => 1 }, check_ssl => { optional => 1 }, nodes => { optional => 1 }, @@ -420,7 +421,7 @@ sub purestorage_volume_connection { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; - my $hname = PVE::INotify::nodename() . "-" . $scfg->{ hgsuffix }; + my $hname = pure_host($scfg); my $params = "host_names=$hname&volume_names=$vgname/$volname"; my $response = $class->purestorage_request( $scfg, "connections", $action, $params ); @@ -470,7 +471,7 @@ sub purestorage_create_volume { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; - my $hname = PVE::INotify::nodename() . "-" . $scfg->{ hgsuffix }; + my $hname = pure_host($scfg); my $params; my $volparams; @@ -508,7 +509,7 @@ sub purestorage_remove_volume { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; - my $hname = PVE::INotify::nodename() . "-" . $scfg->{ hgsuffix }; + my $hname = pure_host($scfg); my ( undef, undef, $vmid ) = $class->parse_volname( $volname ); my $params; @@ -589,7 +590,7 @@ sub purestorage_resize_volume { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; - my $hname = PVE::INotify::nodename() . "-" . $scfg->{ hgsuffix }; + my $hname = pure_host($scfg); my ( $path, undef, undef, $wwid ) = $class->filesystem_path( $scfg, $volname ); my $params = "names=$vgname/$volname"; my $volparams = { "provisioned" => $size }; @@ -1164,4 +1165,14 @@ sub volume_has_feature { return 1 if $features->{ $feature }->{ $key }; return undef; } + +sub pure_host { + my ( $scfg ) = @_; + my $hname = PVE::INotify::nodename(); + my $hgsuffix = $scfg->{ hgsuffix } // $default_hgsuffix; + if ($hgsuffix ne "") { + $hname .= "-" . $hgsuffix; + } + return $hname; +} 1; From 6e184b88e0f7897fab76739efc402fd4bcecf99e Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Sat, 28 Dec 2024 20:10:07 -0500 Subject: [PATCH 02/30] Filter fix in purestorage_list_volumes "qm rescan" was re-adding previously destroyed disks. --- PureStoragePlugin.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index d73ff7e..87d052e 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -295,9 +295,9 @@ sub purestorage_list_volumes { my $filter; if ( defined( $vmid ) ) { - $filter = "name='$vgname/vm-$vmid-disk-*'"; + $filter = "(name='$vgname/vm-$vmid-disk-*'"; $filter .= " or name='$vgname/vm-$vmid-cloudinit'"; - $filter .= " or name='$vgname/vm-$vmid-state-*'"; + $filter .= " or name='$vgname/vm-$vmid-state-*')"; } else { $filter = "name='$vgname/*'"; } From 5ff29eed5b30f2e93a1df6f4a7ec2d0648b4303c Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Mon, 30 Dec 2024 15:25:59 -0500 Subject: [PATCH 03/30] Correct logging output in deactivate_volume() deactivate_volume() should not say "... added to host". --- PureStoragePlugin.pm | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 87d052e..773f211 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -455,10 +455,8 @@ sub purestorage_volume_connection { } } - if ( $action eq "DELETE" ) { - print "Info :: Volume \"$vgname/$volname\" removed from host \"$hname\".\n"; - } - print "Info :: Volume \"$vgname/$volname\" added to host \"$hname\".\n"; + $action = $action eq "DELETE" ? "removed from" : "added to"; + print "Info :: Volume \"$vgname/$volname\" $action host \"$hname\".\n"; return 1; } From ccaad9446efb0dfb9993880018585e13330323f3 Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Mon, 30 Dec 2024 16:25:48 -0500 Subject: [PATCH 04/30] Unmap volume before dropping its connection to storage --- PureStoragePlugin.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 773f211..2fcaa89 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -1060,8 +1060,9 @@ sub deactivate_volume { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; - $class->purestorage_volume_connection( $scfg, $volname, 'DELETE' ); $class->unmap_volume( $storeid, $scfg, $volname, $snapname ); + + $class->purestorage_volume_connection( $scfg, $volname, 'DELETE' ); print "Info :: Volume \"$vgname/$volname\" deactivated.\n"; From ea4021a8d015927e3b1cfe1c66cf5c4e1471656d Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Sat, 4 Jan 2025 09:49:20 -0500 Subject: [PATCH 05/30] alloc_image(): Adjust image size to be at least 1024kb --- PureStoragePlugin.pm | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 2fcaa89..85ccec3 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -823,9 +823,6 @@ sub alloc_image { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; - # Convert size from KB to bytes - my $sizeB = $size * 1024; # KB => B - # Check for supported format (only 'raw' is allowed) die "Error :: Unsupported format ($fmt).\n" if $fmt ne 'raw'; @@ -835,7 +832,13 @@ sub alloc_image { $name = $class->find_free_diskname( $storeid, $scfg, $vmid ) if !$name; # Check size (must be between 1MB and 4PB) - die "Error :: Invalid size ($size kb < 1024 kb).\n" unless $size > 1024; # Proxmox 1MB = 1049KB + if ($size < 1024) { + print "Info :: Size is too small ($size kb), adjusting to 1024 kb\n"; + $size = 1024; + } + + # Convert size from KB to bytes + my $sizeB = $size * 1024; # KB => B if ( !$class->purestorage_create_volume( $scfg, $name, $sizeB, $storeid ) ) { warn "Error :: Failed to create volume \"$vgname/$name\".\n"; From ad11226845bf029913d357cb38ba08b7c23ad28b Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Sun, 5 Jan 2025 18:46:52 -0500 Subject: [PATCH 06/30] Removed unused $disk_prefix variable --- PureStoragePlugin.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 85ccec3..4b1e0d9 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -810,7 +810,6 @@ sub find_free_diskname { my ( $class, $storeid, $scfg, $vmid, $fmt, $add_fmt_suffix ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::find_free_diskname\n" if $DEBUG; - my $disk_prefix = "$scfg->{vgname}/vm-$vmid-disk-"; my $volumes = $class->purestorage_list_volumes( $scfg, $vmid, $storeid ); my @disk_list = map { $_->{ name } } @$volumes; From 78a67d852cbdbe3cb4d379e68a78f9485f3e5d31 Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Sun, 5 Jan 2025 19:26:41 -0500 Subject: [PATCH 07/30] Added purestorage_list_volumes2() allowing to filter by name(s) --- PureStoragePlugin.pm | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 4b1e0d9..358d46b 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -289,21 +289,25 @@ sub purestorage_volume_info { sub purestorage_list_volumes { my ( $class, $scfg, $vmid, $storeid, $destroyed ) = @_; + + my $names = defined ($vmid) ? "vm-$vmid-disk-*,vm-$vmid-cloudinit,vm-$vmid-state-*" : "*"; + + return $class->purestorage_list_volumes2( $scfg, $names, $storeid, $destroyed ); +} + +sub purestorage_list_volumes2 { + my ( $class, $scfg, $names, $storeid, $destroyed ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_list_volumes\n" if $DEBUG; my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; $class->assert_multipath_support(); - my $filter; - if ( defined( $vmid ) ) { - $filter = "(name='$vgname/vm-$vmid-disk-*'"; - $filter .= " or name='$vgname/vm-$vmid-cloudinit'"; - $filter .= " or name='$vgname/vm-$vmid-state-*')"; - } else { - $filter = "name='$vgname/*'"; - } + my @names_list = map { "name='$vgname/$_'" } split( ',', $names ); + + my $filter = join( ' or ', @names_list ); if ( defined( $destroyed ) ) { - $filter .= $destroyed ? " and destroyed='true'" : " and destroyed='false'"; + $filter = '(' . $filter . ')' if $#names_list > 0; + $filter .= " and destroyed='" . ( $destroyed ? "true" : "false" ) . "'"; } my $response = $class->purestorage_request( $scfg, "volumes", "GET", "filter=" . uri_escape( $filter ) ); From 3ce60e55e491bc08549d0bcb67bec6d9521f4fed Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Sun, 5 Jan 2025 19:45:12 -0500 Subject: [PATCH 08/30] Simplified purestorage_get_wwn() Use purestorage_list_volumes2() to return information for the specified volume name. Exclude destroyed volumes to never return a serial number of a previously destroyed volume with the same name. --- PureStoragePlugin.pm | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 358d46b..ed664d8 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -346,17 +346,13 @@ sub purestorage_get_wwn { my ( $class, $scfg, $volname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_get_wwn\n" if $DEBUG; - my ( $vtype, $name, $vmid ) = $class->parse_volname( $volname ); - my $volumes = $class->purestorage_list_volumes( $scfg, $vmid ); + my $volumes = $class->purestorage_list_volumes2( $scfg, $volname, undef, 0 ); foreach my $volume ( @$volumes ) { - if ( $volume->{ name } =~ /vm-$vmid-$name/ ) { - - # Construct the WWN path - my $path = lc( "/dev/disk/by-id/wwn-0x" . $purestorage_wwn_prefix . $volume->{ serial } ); - my $wwn = lc( "3" . $purestorage_wwn_prefix . $volume->{ serial } ); - return ( $path, $vmid, $vtype, $wwn ); - } + # Construct the WWN path + my $path = lc( "/dev/disk/by-id/wwn-0x" . $purestorage_wwn_prefix . $volume->{ serial } ); + my $wwn = lc( "3" . $purestorage_wwn_prefix . $volume->{ serial } ); + return ( $path, $wwn ); } return 0; @@ -789,7 +785,10 @@ sub filesystem_path { my ( $class, $scfg, $volname, $snapname ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::filesystem_path\n" if $DEBUG; - my ( $path, $vmid, $vtype, $wwid ) = $class->purestorage_get_wwn( $scfg, $volname ); + # do we even need this? + my ( $vtype, undef, $vmid ) = $class->parse_volname( $volname ); + + my ( $path, $wwid ) = $class->purestorage_get_wwn( $scfg, $volname ); if ( !defined( $path ) || !defined( $vmid ) || !defined( $vtype ) ) { return wantarray ? ( "", "", "", "" ) : ""; From 3dc6baaa0cbe0d63be29cfedd66fb70c661a123c Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Mon, 6 Jan 2025 21:06:00 -0500 Subject: [PATCH 09/30] multipath-related clean up Sew issue #1. --- PureStoragePlugin.pm | 48 +++++++++++++------------------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index ed664d8..9dabeac 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -57,19 +57,15 @@ sub assert_iscsi_support { } $cmd->{ "multipath" } = "/sbin/multipath"; -$cmd->{ "multipathd" } = "/sbin/multipath"; -my $found_multipath_support; sub assert_multipath_support { my ( $class, $noerr ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::assert_multipath_support\n" if $DEBUG; - return $found_multipath_support if $found_multipath_support; # assume it won't be removed if ever found $class->assert_iscsi_support(); my $found_multipath_exe = -x $cmd->{ "multipath" }; - my $found_multipathd_exe = -x $cmd->{ "multipathd" }; - if ( $found_multipath_exe && $found_multipathd_exe ) { + if ( $found_multipath_exe ) { return 1; } die "Error :: no multipath support - please install multipath-tools.\n" if !$noerr; @@ -387,10 +383,10 @@ sub purestorage_rescan_diskmap { my ( $class, $path ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_rescan_diskmap\n" if $DEBUG; - eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", "node", "--rescan" ] ) }; + # eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", "node", "--rescan" ] ) }; eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", "session", "--rescan" ] ) }; - eval { run_command( [ $cmd->{ "multipath" }, "-W" ] ) }; - eval { run_command( [ $cmd->{ "multipath" }, "-r", $path ] ) }; + # eval { run_command( [ $cmd->{ "multipath" }, "-W" ] ) }; + # eval { run_command( [ $cmd->{ "multipath" }, "-r", $path ] ) }; return 1; } @@ -945,11 +941,6 @@ sub map_volume { die "Error :: Failed to run 'multipath -a $wwid'. Error :: $@\n"; } - eval { run_command( [ $cmd->{ "multipathd" }, "add", "path", $path ] ); }; - if ( $@ ) { - die "Error :: Failed to run 'multipathd add path $path'. Error :: $@\n"; - } - $class->purestorage_rescan_diskmap( $wwid ); # Wait for the device to apear @@ -975,18 +966,11 @@ sub unmap_volume { print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::unmap_volume\n" if $DEBUG; my ( $path, undef, undef, $wwid ) = $class->filesystem_path( $scfg, $volname ); - my $device_path; - my @slaves = []; - my $slaves_path; - my $slave_name; - my $slave_path; - my $multipath_check = 0; - my $wwid_file = "/etc/multipath/wwids"; if ( $path && -e $path ) { - eval { run_command( [ $cmd->{ "multipathd" }, "disablequeueing", "map", $wwid ] ) }; eval { run_command( [ $cmd->{ "blockdev" }, "--flushbufs", $path ] ) }; + my $device_path; if ( $device_path = readlink( $path ) ) { print "Info :: Device path resolved to \"$device_path\".\n"; } else { @@ -999,6 +983,7 @@ sub unmap_volume { my $multipath_check = `$cmd->{ "multipath" } -l $path`; my $slaves_path = "/sys/block/$device_name/slaves"; + my @slaves = (); if ( -d $slaves_path ) { opendir( my $dh, $slaves_path ) or die "Cannot open directory: $!"; @slaves = grep { !/^\.\.?$/ } readdir( $dh ); @@ -1013,27 +998,22 @@ sub unmap_volume { if ( $multipath_check ) { print "Info :: Device \"$path\" is a multipath device. Proceeding with multipath removal.\n"; - if ( -e $wwid_file ) { - open( my $in, '<', $wwid_file ) or die $!; - open( my $out, '>', $wwid_file ) or die $!; - print $out grep { !/^#3/ } <$in>; - close $in; - close $out; - } - - # If the device is a multipath device, remove the link - eval { run_command( [ $cmd->{ "multipath" }, "-f", $path ] ) == 0 or die "Failed to remove multipath link for \"$path\".\n"; }; - + eval { run_command( [ $cmd->{ "multipath" }, "-w", $wwid ] ) }; if ( $@ ) { - warn "Warning :: $@"; + warn "Warning :: Failed to run 'multipath -w $wwid'. Error :: $@"; } + # remove the link + eval { run_command( [ $cmd->{ "multipath" }, "-f", $path ] ) }; + if ( $@ ) { + warn "Warning :: Failed to run 'multipath -f $path'. Error :: $@"; + } } else { print "Info :: Device \"$path\" is not a multipath device. Skipping multipath removal.\n"; } # Iterate through slaves and delete each device - foreach $slave_name ( @slaves ) { + foreach my $slave_name ( @slaves ) { print "Info :: Remove slave: $slave_name\n" if $DEBUG; if ( $slave_name =~ m|^(sd[a-z]+)$| ) { $slave_name = $1; # untaint; From 3d23f66fd62f62d65d3c9a1b1584cc4f7c3cdc4b Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Tue, 7 Jan 2025 07:28:13 -0500 Subject: [PATCH 10/30] Always try to deactivate volume before removal There are border cases when volume can remain mapped or connected to array even if the vm is not running. Ideally, this should be done at the higher level (Storage.pm?), but the current storage implementation does not do it. --- PureStoragePlugin.pm | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 9dabeac..09fc1b4 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -511,12 +511,6 @@ sub purestorage_remove_volume { $eradicate = ( $volname =~ /^vm-(\d+)-(cloudinit|state-.+)/ ) ? 1 : $eradicate; - my $running = PVE::QemuServer::check_running( $vmid ); - - if ( $running ) { - $class->deactivate_volume( $storeid, $scfg, $volname ); - } - $params = "names=$vgname/$volname"; my $body = { destroyed => \1 }; @@ -853,6 +847,8 @@ sub free_image { my ( $class, $storeid, $scfg, $volname, $isBase ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::free_image\n" if $DEBUG; + $class->deactivate_volume( $storeid, $scfg, $volname ); + $class->purestorage_remove_volume( $scfg, $volname, $storeid ); return undef; From 791f08b69019ae14491effd9fe6f8f959b808fb3 Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Tue, 7 Jan 2025 07:31:48 -0500 Subject: [PATCH 11/30] alloc_image(): Do not try to remove volume if we failed to create it --- PureStoragePlugin.pm | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 09fc1b4..328cd88 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -833,11 +833,7 @@ sub alloc_image { my $sizeB = $size * 1024; # KB => B if ( !$class->purestorage_create_volume( $scfg, $name, $sizeB, $storeid ) ) { - warn "Error :: Failed to create volume \"$vgname/$name\".\n"; - if ( !$class->purestorage_remove_volume( $scfg, $name, $storeid, 1 ) ) { - warn "Error :: Failed to destroy volume \"$vgname/$name\".\n"; - } - die; + die "Error :: Failed to create volume \"$vgname/$name\".\n"; } return $name; From e4f9792f660d1180b370814f0038e103d802c275 Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Tue, 7 Jan 2025 07:36:51 -0500 Subject: [PATCH 12/30] purestorage_create_volume(): minor cleanup and refactoring --- PureStoragePlugin.pm | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 328cd88..cc2d380 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -465,18 +465,10 @@ sub purestorage_create_volume { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; - my $hname = pure_host($scfg); - - my $params; - my $volparams; - my $serial; - my $response; - - $params = "names=$vgname/$volname"; - $volparams = { "provisioned" => $size }; - - $response = $class->purestorage_request( $scfg, "volumes", "POST", $params, $volparams ); + my $params = "names=$vgname/$volname"; + my $volparams = { "provisioned" => $size }; + my $response = $class->purestorage_request( $scfg, "volumes", "POST", $params, $volparams ); if ( $response->{ error } ) { die "Error :: PureStorage API :: Create volume failed.\n" . "=> Trace:\n" @@ -485,8 +477,8 @@ sub purestorage_create_volume { . ( $response->{ content } ? "==> Message: " . Dumper( $response->{ content } ) : "" ); } - $serial = $response->{ content }->{ items }->[0]->{ serial } || die "Error :: Failed to retrieve volume serial"; - print "Info :: Volume \"$vgname/$volname\" created.\n"; + my $serial = $response->{ content }->{ items }->[0]->{ serial } || die "Error :: Failed to retrieve volume serial"; + print "Info :: Volume \"$vgname/$volname\" created (serial=$serial).\n"; return 1; } From 61a60a52038f3084cd7c593d3aeefd4e7f0c8ddd Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Tue, 7 Jan 2025 07:55:57 -0500 Subject: [PATCH 13/30] purestorage_remove_volume(): minor cleanup and frefactoring --- PureStoragePlugin.pm | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index cc2d380..a6304a6 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -487,26 +487,19 @@ sub purestorage_remove_volume { my ( $class, $scfg, $volname, $storeid, $eradicate ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_remove_volume\n" if $DEBUG; - $eradicate //= 0; - - $class->assert_blockdev_support(); - $class->assert_multipath_support(); + if ( $volname =~ /^vm-(\d+)-(cloudinit|state-.+)/ ) { + $eradicate = 1; + } else { + $eradicate //= 0; + } my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; - my $hname = pure_host($scfg); - my ( undef, undef, $vmid ) = $class->parse_volname( $volname ); - - my $params; - my $response; - - $eradicate = ( $volname =~ /^vm-(\d+)-(cloudinit|state-.+)/ ) ? 1 : $eradicate; - - $params = "names=$vgname/$volname"; + my $params = "names=$vgname/$volname"; my $body = { destroyed => \1 }; - $response = $class->purestorage_request( $scfg, "volumes", "PATCH", $params, $body ); + my $response = $class->purestorage_request( $scfg, "volumes", "PATCH", $params, $body ); if ( $response->{ error } ) { if ( $response->{ content }->{ errors }->[0]->{ message } eq "Volume has been deleted." ) { warn "Warning :: PureStorage API :: Destroy volume failed :: Nothing to remove.\n" @@ -527,9 +520,7 @@ sub purestorage_remove_volume { } if ( $eradicate ) { - $params = "names=$vgname/$volname"; - - $response = $class->purestorage_request( $scfg, "volumes", "DELETE", $params, $body ); + $response = $class->purestorage_request( $scfg, "volumes", "DELETE", $params ); if ( $response->{ error } ) { $Data::Dumper::Indent = 0; die "Error :: PureStorage API :: Eradicate volume \"$vgname/$volname\" failed.\n" @@ -542,8 +533,6 @@ sub purestorage_remove_volume { } } - $class->purestorage_cleanup_diskmap(); - return 1; } From d41eddf46a31d9c09f604535db77cbff47609784 Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Tue, 7 Jan 2025 08:02:39 -0500 Subject: [PATCH 14/30] purestorage_resize_volume(): minor clean up and refactoring --- PureStoragePlugin.pm | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index a6304a6..3d484b5 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -559,17 +559,15 @@ sub purestorage_resize_volume { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; - my $hname = pure_host($scfg); - my ( $path, undef, undef, $wwid ) = $class->filesystem_path( $scfg, $volname ); - my $params = "names=$vgname/$volname"; - my $volparams = { "provisioned" => $size }; - my $response = $class->purestorage_request( $scfg, "volumes", "PATCH", $params, $volparams ); - + $scfg->{ cache } ||= {}; $scfg->{ cache }->{ volume_info } ||= {}; $scfg->{ cache }->{ volume_info }->{ "$vgname" } ||= {}; $scfg->{ cache }->{ volume_info }->{ "$vgname" }->{ "$volname" } = {}; + my $params = "names=$vgname/$volname"; + my $volparams = { "provisioned" => $size }; + my $response = $class->purestorage_request( $scfg, "volumes", "PATCH", $params, $volparams ); if ( $response->{ error } ) { die "Error :: PureStorage API :: Resize volume failed.\n" . "=> Trace:\n" @@ -579,6 +577,8 @@ sub purestorage_resize_volume { } print "Info :: Volume \"$vgname/$volname\" resized.\n"; + + my ( $path, undef, undef, $wwid ) = $class->filesystem_path( $scfg, $volname ); $class->purestorage_rescan_diskmap( $wwid ); # Wait for the device size to update @@ -591,14 +591,15 @@ sub purestorage_resize_volume { while ( $iteration < $max_attempts ) { print "Info :: Waiting (" . $iteration . "s) for size update for volume \"$vgname/$volname\"...\n"; - $iteration++; - $new_size = $class->purestorage_get_device_size( $path ); + $new_size = $class->purestorage_get_device_size( $path ); if ( $new_size >= $size ) { print "Info :: New size detected for volume \"$vgname/$volname\": $new_size bytes.\n"; return $new_size; } + sleep $interval; + ++$iteration; } die "Error :: Timeout while waiting for updated size of volume \"$vgname/$volname\".\n"; From 8f09da21615e1b3d01bcbe721b869d3d66b19021 Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Tue, 7 Jan 2025 08:06:41 -0500 Subject: [PATCH 15/30] purestorage_volume_connection(): minor clean up and refactoring Also merged pure_host() into purestorage_volume_connection(). --- PureStoragePlugin.pm | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 3d484b5..ae0c91a 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -417,11 +417,14 @@ sub purestorage_volume_connection { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; - my $hname = pure_host($scfg); + + my $hname = PVE::INotify::nodename(); + my $hgsuffix = $scfg->{ hgsuffix } // $default_hgsuffix; + $hname .= "-" . $hgsuffix if $hgsuffix ne ""; my $params = "host_names=$hname&volume_names=$vgname/$volname"; - my $response = $class->purestorage_request( $scfg, "connections", $action, $params ); + my $response = $class->purestorage_request( $scfg, "connections", $action, $params ); if ( $response->{ error } ) { if ( $response->{ content }->{ errors }->[0]->{ message } eq "Connection already exists." ) { warn "Error :: PureStorage API :: Connections \"$vgname/$volname\" to \"$hname\" already exist.\n" @@ -1123,14 +1126,4 @@ sub volume_has_feature { return 1 if $features->{ $feature }->{ $key }; return undef; } - -sub pure_host { - my ( $scfg ) = @_; - my $hname = PVE::INotify::nodename(); - my $hgsuffix = $scfg->{ hgsuffix } // $default_hgsuffix; - if ($hgsuffix ne "") { - $hname .= "-" . $hgsuffix; - } - return $hname; -} 1; From 3702c5670e2df9936665d9deac8512aa7c5c9856 Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Tue, 7 Jan 2025 08:34:45 -0500 Subject: [PATCH 16/30] purestorage_rescan_diskmap(): rescan node on resize --- PureStoragePlugin.pm | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index ae0c91a..a22de90 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -380,11 +380,11 @@ sub purestorage_unmap_disk { } sub purestorage_rescan_diskmap { - my ( $class, $path ) = @_; + my ( $class, $mode, $path ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_rescan_diskmap\n" if $DEBUG; - # eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", "node", "--rescan" ] ) }; - eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", "session", "--rescan" ] ) }; + eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", $mode, "--rescan" ] ) }; + # eval { run_command( [ $cmd->{ "multipath" }, "-W" ] ) }; # eval { run_command( [ $cmd->{ "multipath" }, "-r", $path ] ) }; @@ -582,7 +582,7 @@ sub purestorage_resize_volume { print "Info :: Volume \"$vgname/$volname\" resized.\n"; my ( $path, undef, undef, $wwid ) = $class->filesystem_path( $scfg, $volname ); - $class->purestorage_rescan_diskmap( $wwid ); + $class->purestorage_rescan_diskmap( 'node', $wwid ); # Wait for the device size to update my $iteration = 0; @@ -918,7 +918,7 @@ sub map_volume { die "Error :: Failed to run 'multipath -a $wwid'. Error :: $@\n"; } - $class->purestorage_rescan_diskmap( $wwid ); + $class->purestorage_rescan_diskmap( 'session', $wwid ); # Wait for the device to apear my $iteration = 0; From b8199a39de8f9054ceebc2d21b6a6dfe67f52f4a Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Tue, 7 Jan 2025 08:56:37 -0500 Subject: [PATCH 17/30] Eliminated purestorage_rescan_diskmap() Call "multipath -r" on volume resize only. --- PureStoragePlugin.pm | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index a22de90..87ca838 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -39,13 +39,11 @@ my $DEBUG = 0; my $cmd = {}; # Initialize as a hash, not an array $cmd->{ "iscsiadm" } = "/usr/bin/iscsiadm"; -my $found_iscsi_adm_support; sub assert_iscsi_support { my ( $class, $noerr ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::assert_iscsi_support\n" if $DEBUG; - return $found_iscsi_adm_support if $found_iscsi_adm_support; # assume it won't be removed if ever found - + my $found_iscsi_adm_exe = -x $cmd->{ "iscsiadm" }; if ( $found_iscsi_adm_exe ) { @@ -379,18 +377,6 @@ sub purestorage_unmap_disk { return 1; } -sub purestorage_rescan_diskmap { - my ( $class, $mode, $path ) = @_; - print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_rescan_diskmap\n" if $DEBUG; - - eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", $mode, "--rescan" ] ) }; - - # eval { run_command( [ $cmd->{ "multipath" }, "-W" ] ) }; - # eval { run_command( [ $cmd->{ "multipath" }, "-r", $path ] ) }; - - return 1; -} - sub purestorage_cleanup_diskmap { my ( $class ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_cleanup_diskmap\n" if $DEBUG; @@ -582,8 +568,18 @@ sub purestorage_resize_volume { print "Info :: Volume \"$vgname/$volname\" resized.\n"; my ( $path, undef, undef, $wwid ) = $class->filesystem_path( $scfg, $volname ); - $class->purestorage_rescan_diskmap( 'node', $wwid ); + + eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", "node", "--rescan" ] ) }; + if ( $@ ) { + die "Error :: Failed to run 'iscsiadm --mode node --rescan' command. Error :: $@\n"; + } + # FIXME: wwid is probably ignored + eval { run_command( [ $cmd->{ "multipath" }, "-r", $wwid ] ) }; + if ( $@ ) { + die "Error :: Cannot execute 'multipath -r $wwid' command. Error :: $@\n"; + } + # Wait for the device size to update my $iteration = 0; my $max_attempts = 15; # Max iter count @@ -918,7 +914,10 @@ sub map_volume { die "Error :: Failed to run 'multipath -a $wwid'. Error :: $@\n"; } - $class->purestorage_rescan_diskmap( 'session', $wwid ); + eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", "session", "--rescan" ] ) }; + if ( $@ ) { + die "Error :: Failed to run 'iscsiadm --node session --rescan' command. Error :: $@\n"; + } # Wait for the device to apear my $iteration = 0; From ac595ce025bba5f11a9ea143c1e4a93b807f68d8 Mon Sep 17 00:00:00 2001 From: dpetrov67 Date: Tue, 7 Jan 2025 15:11:39 -0500 Subject: [PATCH 18/30] purestorage_volume_connection(): be silent on non-fatal errors Instead of reporting errors when volume is already connected to/removed from a host, add "was already" to the info output. --- PureStoragePlugin.pm | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 87ca838..2a79f4d 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -408,29 +408,18 @@ sub purestorage_volume_connection { my $hgsuffix = $scfg->{ hgsuffix } // $default_hgsuffix; $hname .= "-" . $hgsuffix if $hgsuffix ne ""; - my $params = "host_names=$hname&volume_names=$vgname/$volname"; + my $params = "host_names=$hname&volume_names=$vgname/$volname"; my $response = $class->purestorage_request( $scfg, "connections", $action, $params ); + my $message; if ( $response->{ error } ) { - if ( $response->{ content }->{ errors }->[0]->{ message } eq "Connection already exists." ) { - warn "Error :: PureStorage API :: Connections \"$vgname/$volname\" to \"$hname\" already exist.\n" - . "=> Trace:\n" - . "==> Code: " - . $response->{ error } . "\n" - . ( $response->{ content } ? "==> Message: " . Dumper( $response->{ content } ) : "" ); - } elsif ( $response->{ content }->{ errors }->[0]->{ message } eq "Volume has been destroyed." ) { - warn "Error :: PureStorage API :: Failed to modify connection :: Nothing to remove.\n" - . "=> Trace:\n" - . "==> Code: " - . $response->{ error } . "\n" - . ( $response->{ content } ? "==> Message: " . Dumper( $response->{ content } ) : "" ); - } elsif ( $response->{ content }->{ errors }->[0]->{ message } eq "Connection does not exist." ) { - warn "Error :: PureStorage API :: Failed to modify connection :: Nothing to remove.\n" - . "=> Trace:\n" - . "==> Code: " - . $response->{ error } . "\n" - . ( $response->{ content } ? "==> Message: " . Dumper( $response->{ content } ) : "" ); - } else { + $message = $response->{ content }->{ errors }->[0]->{ message } || '*'; + if ( $message eq "Connection already exists." ) { + $message = '' if $action eq 'POST'; + } elsif ( $message eq "Volume has been destroyed." || $message eq "Connection does not exist.") { + $message = '' if $action eq 'DELETE'; + } + if ( $message ne '' ) { $Data::Dumper::Indent = 0; die "Error :: PureStorage API :: Failed to modify connection.\n" . "=> Trace:\n" @@ -438,10 +427,13 @@ sub purestorage_volume_connection { . $response->{ error } . "\n" . ( $response->{ content } ? "==> Message: " . Dumper( $response->{ content } ) : "" ); } + $message = 'was already'; + } else { + $message = 'is'; } - $action = $action eq "DELETE" ? "removed from" : "added to"; - print "Info :: Volume \"$vgname/$volname\" $action host \"$hname\".\n"; + $message .= ' ' . ($action eq 'DELETE' ? 'removed from' : 'added to'); + print "Info :: Volume \"$vgname/$volname\" $message host \"$hname\".\n"; return 1; } From 288300aa1f9f683551bb2387b1c8373bd19d34d7 Mon Sep 17 00:00:00 2001 From: Andreas Steinel Date: Thu, 9 Jan 2025 08:33:29 +0100 Subject: [PATCH 19/30] correct name for storage.cfg The configuration file is not storage.conf, but storage.cfg. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ec95cb9..a43da67 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ sudo apt install ./libpve-storage-purestorage-perl.deb ## Configuration -After installing the plugin, you need to configure Proxmox VE to use it. Since Proxmox VE does not currently support adding custom storage plugins via the GUI, you will need to manually edit the storage configuration file `/etc/pve/storage.conf`. +After installing the plugin, you need to configure Proxmox VE to use it. Since Proxmox VE does not currently support adding custom storage plugins via the GUI, you will need to manually edit the storage configuration file `/etc/pve/storage.cfg`. ``` purestorage: pure From c06e2f24c0d83ef56e1211326b9e72753dcc95b6 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 13 Jan 2025 12:12:12 -0500 Subject: [PATCH 20/30] Add $storeid to cache key in list_images() and status() This allows to have multiple instances of the plugin with different parameters. --- PureStoragePlugin.pm | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 2a79f4d..e46d07f 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -825,47 +825,48 @@ sub free_image { sub list_images { my ( $class, $storeid, $scfg, $vmid, $vollist, $cache ) = @_; - print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::list_images\n" if $DEBUG; - if ( !$cache->{ purestorage } ) { - $cache->{ purestorage } = $class->purestorage_list_volumes( $scfg, $vmid, $storeid, 0 ); - } else { + my $key = type() . ':' . $storeid; + if ( $cache->{ $key } ) { print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::list_images::cached\n" if $DEBUG; + } else { + print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::list_images\n" if $DEBUG; + $cache->{ $key } = $class->purestorage_list_volumes( $scfg, $vmid, $storeid, 0 ); } - return $cache->{ purestorage }; + return $cache->{ $key }; } sub status { my ( $class, $storeid, $scfg, $cache ) = @_; - print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::status\n" if $DEBUG; - $cache->{ purestorage_size } //= {}; - $cache->{ purestorage_size }->{ last_update } //= 0; + $cache = $cache->{ type() . ':' . $storeid } //= {}; + $cache->{ last_update } //= 0; my $current_time = gettimeofday(); - - if ( $current_time - $cache->{ purestorage_size }->{ last_update } >= 60 ) { + if ( $current_time - $cache->{ last_update } >= 60 ) { + print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::status\n" if $DEBUG; + my $response = $class->purestorage_request( $scfg, "arrays/space", "GET" ); # Get storage capacity and used space from the response - $cache->{ purestorage_size }->{ total } = $response->{ content }->{ items }->[0]->{ capacity }; - $cache->{ purestorage_size }->{ used } = $response->{ content }->{ items }->[0]->{ space }->{ total_physical }; - - # $cache->{ purestorage_size }->{ used } = $response->{ content }->{ items }->[0]->{ space }->{ total_used }; # Do not know what is correct - $cache->{ purestorage_size }->{ last_update } = $current_time; + $cache->{ total } = $response->{ content }->{ items }->[0]->{ capacity }; + $cache->{ used } = $response->{ content }->{ items }->[0]->{ space }->{ total_physical }; + # $cache->{ used } = $response->{ content }->{ items }->[0]->{ space }->{ total_used }; # Do not know what is correct + + $cache->{ last_update } = $current_time; } else { print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::status::cached\n" if $DEBUG; } # Calculate free space - my $free = $cache->{ purestorage_size }->{ total } - $cache->{ purestorage_size }->{ used }; + my $free = $cache->{ total } - $cache->{ used }; # Mark storage as active my $active = 1; # Return total, free, used space and the active status - return ( $cache->{ purestorage_size }->{ total }, $free, $cache->{ purestorage_size }->{ used }, $active ); + return ( $cache->{ total }, $free, $cache->{ used }, $active ); } sub activate_storage { From 01d502007c98ade2e67155de43ef67d8d80ae352 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 13 Jan 2025 12:30:26 -0500 Subject: [PATCH 21/30] Show slave disk names on one line in unmap_volume() --- PureStoragePlugin.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index e46d07f..bb9c8cc 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -958,7 +958,7 @@ sub unmap_volume { @slaves = grep { !/^\.\.?$/ } readdir( $dh ); closedir( $dh ); - print "Info :: Disk \"$device_name\" slaves: \n" . Dumper( @slaves ) if $DEBUG; + print "Info :: Disk \"$device_name\" slaves: " . join( ', ', @slaves ) . "\n" if $DEBUG; } elsif ( $device_name =~ m|^(sd[a-z]+)$| ) { warn "Warning :: Disk \"$device_name\" has no slaves.\n"; push @slaves, $1; From 13b66fdfafc2b7d98167e45e1cb40741862f7c44 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 13 Jan 2025 14:51:54 -0500 Subject: [PATCH 22/30] parse_volume(): return volume format as 'raw' This fixes an issue with migration of EFI disks. --- PureStoragePlugin.pm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index bb9c8cc..373a19f 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -738,8 +738,10 @@ sub parse_volname { my $vmid = $2; # Extract VMID my $name = $3; # Remaining part of the volume name - return ( $vtype, $name, $vmid ); # Return type, name, and VMID + # ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) + return ( $vtype, $name, $vmid, undef, undef, undef, 'raw' ); } + die "Error :: Invalid volume name ($volname).\n"; return 0; } From 3f1c997a758b35f63c2d9c0caae3d5b64c0e6e9d Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 13 Jan 2025 15:34:21 -0500 Subject: [PATCH 23/30] Cleanup: removed unneeded assert_* functions --- PureStoragePlugin.pm | 64 +++++--------------------------------------- 1 file changed, 6 insertions(+), 58 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 373a19f..f44fad6 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -35,59 +35,11 @@ my $default_hgsuffix = "pve"; my $DEBUG = 0; -### BLOCK: Asserts - -my $cmd = {}; # Initialize as a hash, not an array -$cmd->{ "iscsiadm" } = "/usr/bin/iscsiadm"; - -sub assert_iscsi_support { - my ( $class, $noerr ) = @_; - print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::assert_iscsi_support\n" if $DEBUG; - - my $found_iscsi_adm_exe = -x $cmd->{ "iscsiadm" }; - - if ( $found_iscsi_adm_exe ) { - return 1; - } - die "Error :: no iSCSI support - please install open-iscsi.\n" if !$noerr; - warn "Warning :: no iSCSI support - please install open-iscsi.\n"; - return 0; -} - -$cmd->{ "multipath" } = "/sbin/multipath"; - -sub assert_multipath_support { - my ( $class, $noerr ) = @_; - print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::assert_multipath_support\n" if $DEBUG; - $class->assert_iscsi_support(); - - my $found_multipath_exe = -x $cmd->{ "multipath" }; - - if ( $found_multipath_exe ) { - return 1; - } - die "Error :: no multipath support - please install multipath-tools.\n" if !$noerr; - warn "Warning :: no multipath support - please install multipath-tools.\n"; - return 0; -} - -$cmd->{ "blockdev" } = "/usr/sbin/blockdev"; -my $found_blockdev_support; - -sub assert_blockdev_support { - my ( $class, $noerr ) = @_; - print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::assert_blockdev_support\n" if $DEBUG; - return $found_blockdev_support if $found_blockdev_support; # assume it won't be removed if ever found - - my $found_blockdev_exe = -x $cmd->{ "blockdev" }; - - if ( $found_blockdev_exe ) { - return 1; - } - die "Error :: no blockdev support - please install blockdev.\n" if !$noerr; - warn "Warning :: no blockdev support - please install blockdev.\n"; - return 0; -} +my $cmd = { + iscsiadm => '/usr/bin/iscsiadm', + multipath => '/sbin/multipath', + blockdev => '/usr/sbin/blockdev' +}; ### BLOCK: Configuration sub api { @@ -293,7 +245,6 @@ sub purestorage_list_volumes2 { my ( $class, $scfg, $names, $storeid, $destroyed ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_list_volumes\n" if $DEBUG; my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; - $class->assert_multipath_support(); my @names_list = map { "name='$vgname/$_'" } split( ',', $names ); @@ -305,7 +256,6 @@ sub purestorage_list_volumes2 { } my $response = $class->purestorage_request( $scfg, "volumes", "GET", "filter=" . uri_escape( $filter ) ); - if ( $response->{ error } ) { die "Error :: PureStorage API :: List volumes status failed.\n" . "=> Trace:\n" @@ -329,7 +279,7 @@ sub purestorage_list_volumes2 { size => $_->{ provisioned }, ctime => $ctime, volid => $storeid ? "$storeid:$volname" : $volname, - format => "raw" + format => 'raw' } } @{ $response->{ content }->{ items } }; @@ -441,8 +391,6 @@ sub purestorage_create_volume { my ( $class, $scfg, $volname, $size, $storeid ) = @_; print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_create_volume\n" if $DEBUG; - $class->assert_multipath_support(); - my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; From 4484ff1a0183b87a48aa8224b60547907577e348 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 13 Jan 2025 20:03:35 -0500 Subject: [PATCH 24/30] Simplified rename_volume() --- PureStoragePlugin.pm | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index f44fad6..5a23531 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -986,20 +986,15 @@ sub volume_resize { } sub rename_volume { - print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::rename_volume\n"; my ( $class, $scfg, $storeid, $source_volname, $target_vmid, $target_volname ) = @_; + print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::rename_volume\n" if $DEBUG; die "Error :: not implemented in storage plugin \"$class\".\n" if $class->can( 'api' ) && $class->api() < 10; - my ( undef, $source_image, $source_vmid, $base_name, $base_vmid, undef, $format ) = $class->parse_volname( $source_volname ); - - $target_volname = $class->find_free_diskname( $storeid, $scfg, $target_vmid, $format, 1 ) - if !$target_volname; + $target_volname = $class->find_free_diskname( $storeid, $scfg, $target_vmid ) if !$target_volname; $class->purestorage_rename_volume( $scfg, $source_volname, $target_volname ); - $base_name = $base_name ? "${base_name}/" : ''; - - return "${storeid}:${base_name}${target_volname}"; + return "$storeid:$target_volname"; } sub volume_import { From a8de2749c5921c3f3d4a3bdb651fc084f8c2f5b0 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Tue, 14 Jan 2025 11:43:40 -0500 Subject: [PATCH 25/30] unmap_volume(): Use $wwid instead of $path where possible Other changes: * Replaced readlink() with abs_path() * Added error check for blockdev call --- PureStoragePlugin.pm | 115 +++++++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 59 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 5a23531..9d261d8 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -22,6 +22,7 @@ use HTTP::Request (); use URI::Escape qw( uri_escape ); use File::Basename qw( basename ); use Time::HiRes qw( gettimeofday sleep ); +use Cwd qw( abs_path ); use base qw(PVE::Storage::Plugin); @@ -886,66 +887,62 @@ sub unmap_volume { my ( $path, undef, undef, $wwid ) = $class->filesystem_path( $scfg, $volname ); - if ( $path && -e $path ) { - eval { run_command( [ $cmd->{ "blockdev" }, "--flushbufs", $path ] ) }; - - my $device_path; - if ( $device_path = readlink( $path ) ) { - print "Info :: Device path resolved to \"$device_path\".\n"; - } else { - die "Error :: unable to read device link.\n"; - } - - my $device_name = basename( $device_path ); - print "Info :: Device name resolved to \"$device_name\".\n"; - - my $multipath_check = `$cmd->{ "multipath" } -l $path`; - my $slaves_path = "/sys/block/$device_name/slaves"; - - my @slaves = (); - if ( -d $slaves_path ) { - opendir( my $dh, $slaves_path ) or die "Cannot open directory: $!"; - @slaves = grep { !/^\.\.?$/ } readdir( $dh ); - closedir( $dh ); - - print "Info :: Disk \"$device_name\" slaves: " . join( ', ', @slaves ) . "\n" if $DEBUG; - } elsif ( $device_name =~ m|^(sd[a-z]+)$| ) { - warn "Warning :: Disk \"$device_name\" has no slaves.\n"; - push @slaves, $1; - } - - if ( $multipath_check ) { - print "Info :: Device \"$path\" is a multipath device. Proceeding with multipath removal.\n"; - - eval { run_command( [ $cmd->{ "multipath" }, "-w", $wwid ] ) }; - if ( $@ ) { - warn "Warning :: Failed to run 'multipath -w $wwid'. Error :: $@"; - } - - # remove the link - eval { run_command( [ $cmd->{ "multipath" }, "-f", $path ] ) }; - if ( $@ ) { - warn "Warning :: Failed to run 'multipath -f $path'. Error :: $@"; - } - } else { - print "Info :: Device \"$path\" is not a multipath device. Skipping multipath removal.\n"; - } - - # Iterate through slaves and delete each device - foreach my $slave_name ( @slaves ) { - print "Info :: Remove slave: $slave_name\n" if $DEBUG; - if ( $slave_name =~ m|^(sd[a-z]+)$| ) { - $slave_name = $1; # untaint; - $class->purestorage_unmap_disk( $slave_name ); - } else { - die "Error :: Invalid disk name \"$slave_name\"."; - } - } - - print "Info :: Device \"$device_name\" removed from system.\n"; - return 1; + my $device_path = abs_path( $path ); + if ( defined($device_path) && -b $device_path ) { + print "Info :: Device path resolved to \"$device_path\".\n"; + } else { + die "Error :: unable to get device path for $path - $!.\n"; } - return 0; + + eval { run_command( [ $cmd->{ "blockdev" }, "--flushbufs", $path ] ) }; + if ( $@ ) { + warn "Warning :: Failed to run 'blockdev --flushbufs $path'. Error :: $@"; + } + + my $device_name = basename( $device_path ); + my $slaves_path = "/sys/block/$device_name/slaves"; + + my @slaves = (); + if ( -d $slaves_path ) { + opendir( my $dh, $slaves_path ) or die "Cannot open directory: $!"; + @slaves = grep { !/^\.\.?$/ } readdir( $dh ); + closedir( $dh ); + print "Info :: Disk \"$device_name\" slaves: " . join( ', ', @slaves ) . "\n" if $DEBUG; + } elsif ( $device_name =~ m|^(sd[a-z]+)$| ) { + warn "Warning :: Disk \"$device_name\" has no slaves.\n"; + push @slaves, $1; + } + + my $multipath_check = `$cmd->{ "multipath" } -l $wwid`; + if ( $multipath_check ) { + print "Info :: Device \"$device_path\" is a multipath device. Proceeding with multipath removal.\n"; + eval { run_command( [ $cmd->{ "multipath" }, "-w", $wwid ] ) }; + if ( $@ ) { + warn "Warning :: Failed to run 'multipath -w $wwid'. Error :: $@"; + } + + # remove the link + eval { run_command( [ $cmd->{ "multipath" }, "-f", $wwid ] ) }; + if ( $@ ) { + warn "Warning :: Failed to run 'multipath -f $wwid'. Error :: $@"; + } + } else { + print "Info :: Device \"$wwid\" is not a multipath device. Skipping multipath removal.\n"; + } + + # Iterate through slaves and delete each device + foreach my $slave_name ( @slaves ) { + print "Info :: Remove slave: $slave_name\n" if $DEBUG; + if ( $slave_name =~ m|^(sd[a-z]+)$| ) { + $slave_name = $1; # untaint; + $class->purestorage_unmap_disk( $slave_name ); + } else { + die "Error :: Invalid disk name \"$slave_name\"."; + } + } + + print "Info :: Device \"$device_name\" removed from system.\n"; + return 1; } sub activate_volume { From d2fecfd95a6655f8e48bcbae79fab0f1abd0d8be Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Tue, 14 Jan 2025 13:35:00 -0500 Subject: [PATCH 26/30] unmap_volume(): Do not die if volume is already unmapped (accidentally reintroduced the issue in prior commit) --- PureStoragePlugin.pm | 110 ++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 9d261d8..656039b 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -887,62 +887,66 @@ sub unmap_volume { my ( $path, undef, undef, $wwid ) = $class->filesystem_path( $scfg, $volname ); - my $device_path = abs_path( $path ); - if ( defined($device_path) && -b $device_path ) { - print "Info :: Device path resolved to \"$device_path\".\n"; - } else { - die "Error :: unable to get device path for $path - $!.\n"; - } - - eval { run_command( [ $cmd->{ "blockdev" }, "--flushbufs", $path ] ) }; - if ( $@ ) { - warn "Warning :: Failed to run 'blockdev --flushbufs $path'. Error :: $@"; - } - - my $device_name = basename( $device_path ); - my $slaves_path = "/sys/block/$device_name/slaves"; - - my @slaves = (); - if ( -d $slaves_path ) { - opendir( my $dh, $slaves_path ) or die "Cannot open directory: $!"; - @slaves = grep { !/^\.\.?$/ } readdir( $dh ); - closedir( $dh ); - print "Info :: Disk \"$device_name\" slaves: " . join( ', ', @slaves ) . "\n" if $DEBUG; - } elsif ( $device_name =~ m|^(sd[a-z]+)$| ) { - warn "Warning :: Disk \"$device_name\" has no slaves.\n"; - push @slaves, $1; - } - - my $multipath_check = `$cmd->{ "multipath" } -l $wwid`; - if ( $multipath_check ) { - print "Info :: Device \"$device_path\" is a multipath device. Proceeding with multipath removal.\n"; - eval { run_command( [ $cmd->{ "multipath" }, "-w", $wwid ] ) }; - if ( $@ ) { - warn "Warning :: Failed to run 'multipath -w $wwid'. Error :: $@"; - } - - # remove the link - eval { run_command( [ $cmd->{ "multipath" }, "-f", $wwid ] ) }; - if ( $@ ) { - warn "Warning :: Failed to run 'multipath -f $wwid'. Error :: $@"; - } - } else { - print "Info :: Device \"$wwid\" is not a multipath device. Skipping multipath removal.\n"; - } - - # Iterate through slaves and delete each device - foreach my $slave_name ( @slaves ) { - print "Info :: Remove slave: $slave_name\n" if $DEBUG; - if ( $slave_name =~ m|^(sd[a-z]+)$| ) { - $slave_name = $1; # untaint; - $class->purestorage_unmap_disk( $slave_name ); + if ( $path && -b $path ) { + my $device_path = abs_path( $path ); + if ( defined( $device_path ) ) { + print "Info :: Device path resolved to \"$device_path\".\n"; } else { - die "Error :: Invalid disk name \"$slave_name\"."; + die "Error :: unable to get device path for $path - $!.\n"; + } + + eval { run_command( [ $cmd->{ "blockdev" }, "--flushbufs", $path ] ) }; + if ( $@ ) { + warn "Warning :: Failed to run 'blockdev --flushbufs $path'. Error :: $@"; } - } - print "Info :: Device \"$device_name\" removed from system.\n"; - return 1; + my $device_name = basename( $device_path ); + my $slaves_path = "/sys/block/$device_name/slaves"; + + my @slaves = (); + if ( -d $slaves_path ) { + opendir( my $dh, $slaves_path ) or die "Cannot open directory: $!"; + @slaves = grep { !/^\.\.?$/ } readdir( $dh ); + closedir( $dh ); + print "Info :: Disk \"$device_name\" slaves: " . join( ', ', @slaves ) . "\n" if $DEBUG; + } elsif ( $device_name =~ m|^(sd[a-z]+)$| ) { + warn "Warning :: Disk \"$device_name\" has no slaves.\n"; + push @slaves, $1; + } + + my $multipath_check = `$cmd->{ "multipath" } -l $wwid`; + if ( $multipath_check ) { + print "Info :: Device \"$device_path\" is a multipath device. Proceeding with multipath removal.\n"; + eval { run_command( [ $cmd->{ "multipath" }, "-w", $wwid ] ) }; + if ( $@ ) { + warn "Warning :: Failed to run 'multipath -w $wwid'. Error :: $@"; + } + + # remove the link + eval { run_command( [ $cmd->{ "multipath" }, "-f", $wwid ] ) }; + if ( $@ ) { + warn "Warning :: Failed to run 'multipath -f $wwid'. Error :: $@"; + } + } else { + print "Info :: Device \"$wwid\" is not a multipath device. Skipping multipath removal.\n"; + } + + # Iterate through slaves and delete each device + foreach my $slave_name ( @slaves ) { + print "Info :: Remove slave: $slave_name\n" if $DEBUG; + if ( $slave_name =~ m|^(sd[a-z]+)$| ) { + $slave_name = $1; # untaint; + $class->purestorage_unmap_disk( $slave_name ); + } else { + die "Error :: Invalid disk name \"$slave_name\"."; + } + } + + print "Info :: Device \"$device_name\" removed from system.\n"; + return 1; + } + + return 0; } sub activate_volume { From 16721e35d1287714d69ad1feaadc7131fb04afda Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Tue, 14 Jan 2025 14:32:45 -0500 Subject: [PATCH 27/30] Updated recommended multipah.conf file Added Pure wwids to blacklist_exceptions section. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a43da67..4c98995 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ blacklist { } blacklist_exceptions { + wwid "624a9370.*" device { vendor "PURE" } From 246cba516e619bd86a9d5503500f586ffa7eb454 Mon Sep 17 00:00:00 2001 From: amulet1 Date: Wed, 15 Jan 2025 12:27:35 -0500 Subject: [PATCH 28/30] Added wrapper to run_command() to streamline error handling Replaced run_command() occurences with exec_command() which takes care of errors Setting $DEBUG set to 2+ will show shell commands we are executing in the log --- PureStoragePlugin.pm | 79 ++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/PureStoragePlugin.pm b/PureStoragePlugin.pm index 656039b..e863bbe 100644 --- a/PureStoragePlugin.pm +++ b/PureStoragePlugin.pm @@ -109,6 +109,21 @@ sub options { }; } +### BLOCK: Supporting functions + +sub exec_command { + my ( $command, $die, %param ) = @_; + + print "Debug :: execute '" . join( ' ', @$command ) . "'\n" if $DEBUG >= 2; + eval { run_command( $command, %param ) }; + if ( $@ ) { + my $error = " :: Cannot execute '" . join( ' ', @$command ) . "'. Error :: $@\n"; + die 'Error' . $error if $die; + + warn 'Warning' . $error; + } +} + ### BLOCK: Local multipath => PVE::Storage::Custom::PureStoragePlugin::sub::s sub purestorage_request { @@ -313,7 +328,7 @@ sub purestorage_unmap_disk { my $disk_path = "/dev/$disk_name"; if ( -e $disk_path ) { - run_command( [ $cmd->{ "blockdev" }, "--flushbufs", $disk_path ] ); + exec_command( [ $cmd->{ blockdev }, '--flushbufs', $disk_path ] ); } my $fh; @@ -471,15 +486,14 @@ sub purestorage_get_device_size { print "Debug :: PVE::Storage::Custom::PureStoragePlugin::sub::purestorage_get_device_size\n" if $DEBUG; my $size = 0; - eval { - run_command( [ $cmd->{ "blockdev" }, "--getsize64", $path ], outfunc => sub { $size = $_[0]; } ); - }; - if ( $@ ) { - die "Error :: Cannot execute 'blockdev' command for \"$path\". Error :: $@\n"; - } + exec_command( [ $cmd->{ blockdev }, '--getsize64', $path ], 1, + outfunc => sub { + $size = $_[0]; + chomp $size; + } + ); print "Debug :: Detected size: $size\n" if $DEBUG; - chomp $size; return $size; } @@ -489,7 +503,7 @@ sub purestorage_resize_volume { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; my $url = $scfg->{ address } || die "Error :: Pure Storage host is not defined.\n"; - + $scfg->{ cache } ||= {}; $scfg->{ cache }->{ volume_info } ||= {}; $scfg->{ cache }->{ volume_info }->{ "$vgname" } ||= {}; @@ -507,20 +521,14 @@ sub purestorage_resize_volume { } print "Info :: Volume \"$vgname/$volname\" resized.\n"; - + my ( $path, undef, undef, $wwid ) = $class->filesystem_path( $scfg, $volname ); - - eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", "node", "--rescan" ] ) }; - if ( $@ ) { - die "Error :: Failed to run 'iscsiadm --mode node --rescan' command. Error :: $@\n"; - } + + exec_command( [ $cmd->{ iscsiadm }, '--mode', 'node', '--rescan' ], 1); # FIXME: wwid is probably ignored - eval { run_command( [ $cmd->{ "multipath" }, "-r", $wwid ] ) }; - if ( $@ ) { - die "Error :: Cannot execute 'multipath -r $wwid' command. Error :: $@\n"; - } - + exec_command( [ $cmd->{ multipath }, '-r', $wwid ], 1 ); + # Wait for the device size to update my $iteration = 0; my $max_attempts = 15; # Max iter count @@ -851,17 +859,11 @@ sub map_volume { my $vgname = $scfg->{ vgname } || die "Error :: Volume group name is not defined.\n"; - print "Info :: Mapped volume \"$vgname/$volname\" with WWN: " . uc( $wwid ) . ".\n" if $DEBUG; + print "Info :: Mapping volume \"$vgname/$volname\" with WWN: " . uc( $wwid ) . ".\n" if $DEBUG; - eval { run_command( [ $cmd->{ "multipath" }, "-a", $wwid ] ); }; - if ( $@ ) { - die "Error :: Failed to run 'multipath -a $wwid'. Error :: $@\n"; - } + exec_command( [ $cmd->{ multipath }, '-a', $wwid ], 1 ); - eval { run_command( [ $cmd->{ "iscsiadm" }, "--mode", "session", "--rescan" ] ) }; - if ( $@ ) { - die "Error :: Failed to run 'iscsiadm --node session --rescan' command. Error :: $@\n"; - } + exec_command( [ $cmd->{ iscsiadm }, '--mode', 'session', '--rescan' ], 1 ); # Wait for the device to apear my $iteration = 0; @@ -877,7 +879,7 @@ sub map_volume { sleep $interval; } - warn "Warning :: Local path \"$path\" not exists.\n"; + warn "Warning :: Local path \"$path\" does not exist.\n"; return 0; } @@ -894,11 +896,8 @@ sub unmap_volume { } else { die "Error :: unable to get device path for $path - $!.\n"; } - - eval { run_command( [ $cmd->{ "blockdev" }, "--flushbufs", $path ] ) }; - if ( $@ ) { - warn "Warning :: Failed to run 'blockdev --flushbufs $path'. Error :: $@"; - } + + exec_command( [ $cmd->{ blockdev }, '--flushbufs', $path ] ); my $device_name = basename( $device_path ); my $slaves_path = "/sys/block/$device_name/slaves"; @@ -917,16 +916,10 @@ sub unmap_volume { my $multipath_check = `$cmd->{ "multipath" } -l $wwid`; if ( $multipath_check ) { print "Info :: Device \"$device_path\" is a multipath device. Proceeding with multipath removal.\n"; - eval { run_command( [ $cmd->{ "multipath" }, "-w", $wwid ] ) }; - if ( $@ ) { - warn "Warning :: Failed to run 'multipath -w $wwid'. Error :: $@"; - } + exec_command( [ $cmd->{ multipath }, '-w', $wwid ] ); # remove the link - eval { run_command( [ $cmd->{ "multipath" }, "-f", $wwid ] ) }; - if ( $@ ) { - warn "Warning :: Failed to run 'multipath -f $wwid'. Error :: $@"; - } + exec_command( [ $cmd->{ multipath }, '-f', $wwid ] ); } else { print "Info :: Device \"$wwid\" is not a multipath device. Skipping multipath removal.\n"; } From e4aae03d5111b88abd0ee1675be81006c22b9092 Mon Sep 17 00:00:00 2001 From: timansky Date: Thu, 16 Jan 2025 08:59:08 +0500 Subject: [PATCH 29/30] README added lvm case --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 296b36d..b5a102e 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,18 @@ sudo systemctl restart pve-cluster.service pvedaemon.service pvestatd.service pv - API Token Permissions: Ensure that the API token used has the necessary permissions to create and manage volumes on the Pure Storage array. - Plugin Updates: Ensure you are using the latest version of the plugin. Check the GitHub repository for updates. +### Known issues + +- `lvm inside a volume`: If you plan to use LVM inside a volume, it is better to add purestorage volumes to the ignore list to avoid scanning. + +```bash +cat /etc/lvm/lvmlocal.conf +... +devices { + global_filter=["r|/dev/zd.*|","r|/dev/rbd.*|","r|/dev/mapper/3624a9370.*|"] +} +``` + ## Contributing Contributions to this project are welcome. From 6e68385d9602ff8410713fb6bb975134ee135ca9 Mon Sep 17 00:00:00 2001 From: timansky Date: Thu, 16 Jan 2025 21:56:45 +0500 Subject: [PATCH 30/30] remove zigmund from codeowners --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index ebe02fc..9df087d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @timansky @zigmund +* @timansky