update version to 8.3.6-1

* enable snapshot link clone on zfspool.

Makefile

@@ -18,7 +18,7 @@ $(BUILDDIR):
 	rm -rf $@ $@.tmp
 	cp -a src $@.tmp
 	cp -a debian $@.tmp/
-	echo "git clone git://git.proxmox.com/git/pve-storage.git\\ngit checkout $(GITVERSION)" >$@.tmp/debian/SOURCE
+	echo "git clone https://github.com/jiangcuo/pve-storage.git\\ngit checkout $(GITVERSION)" >$@.tmp/debian/SOURCE
 	mv $@.tmp $@
 
 .PHONY: deb

debian/changelog

@@ -1,3 +1,9 @@
+libpve-storage-perl (8.3.6-1) bookworm; urgency=medium
+
+  * pvebcache: fix issue
+
+ -- Lierfang Support Team <itsupport@lierfang.com>  Mon, 14 Apr 2025 18:58:32 +0800
+
 libpve-storage-perl (8.3.6) bookworm; urgency=medium
 
   * plugin: file size info: be consistent about size of directory subvol to
@@ -35,6 +41,28 @@ libpve-storage-perl (8.3.5) bookworm; urgency=medium
 
  -- Proxmox Support Team <support@proxmox.com>  Sun, 06 Apr 2025 21:18:38 +0200
 
+libpve-storage-perl (8.3.4-3) bookworm; urgency=medium
+
+  * fix bcache syntax error
+
+ -- Lierfang Support Team <itsupport@lierfang.com>  Sun, 09 Mar 2025 17:19:59 +0800
+
+libpve-storage-perl (8.3.4-2) bookworm; urgency=medium
+
+  * fix bcache cli missing
+
+ -- Lierfang Support Team <itsupport@lierfang.com>  Sun, 09 Mar 2025 16:46:16 +0800
+
+libpve-storage-perl (8.3.4-1) bookworm; urgency=medium
+
+  * fix bcache missing.
+
+  * cli: add pvebcache.
+
+  * copyright: add lierfang information.
+
+ -- Lierfang Support Team <itsupport@lierfang.com>  Wed, 26 Feb 2025 17:23:42 +0800
+
 libpve-storage-perl (8.3.4) bookworm; urgency=medium
 
   * rbd plugin: drop broken cache for pool specific information in list image.
@@ -62,6 +90,14 @@ libpve-storage-perl (8.3.4) bookworm; urgency=medium
 
  -- Proxmox Support Team <support@proxmox.com>  Thu, 03 Apr 2025 19:20:17 +0200
 
+libpve-storage-perl (8.3.3+port1) bookworm; urgency=medium
+
+  * add clone_image_pxvirt function for pxvdi
+
+  * Add bcache support
+
+ -- Jiangcuo <jiangcuo@lierfang.com>  Sat, 22 Feb 2025 14:04:30 +0800
+
 libpve-storage-perl (8.3.3) bookworm; urgency=medium
 
   * plugin: export/import: fix calls to path() method

debian/control

@@ -1,7 +1,7 @@
 Source: libpve-storage-perl
 Section: perl
 Priority: optional
-Maintainer: Proxmox Support Team <support@proxmox.com>
+Maintainer: Lierfang Support Team <itsupport@lierfang.com>
 Build-Depends: debhelper-compat (= 13),
                libfile-chdir-perl,
                libposix-strptime-perl,
@@ -18,7 +18,7 @@ Build-Depends: debhelper-compat (= 13),
                pve-qemu-kvm | qemu-utils,
                zfsutils-linux,
 Standards-Version: 4.6.2
-Homepage: https://www.proxmox.com
+Homepage: https://www.lierfang.com
 
 Package: libpve-storage-perl
 Architecture: all
@@ -27,7 +27,8 @@ Breaks: libpve-guest-common-perl (<< 4.0-3),
         pve-container (<< 3.1-2),
         pve-manager (<< 5.2-12),
         qemu-server (<< 8.3.2),
-Depends: bzip2,
+Depends: bcache-tools,
+         bzip2,
          ceph-common (>= 12.2~),
          ceph-fuse,
          cifs-utils,

debian/copyright

@@ -1,3 +1,21 @@
+Copyright (C) 2011 - 2025 Lierfang 
+
+This software is maintained by Lierfang <itsupport@lierfang.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
 Copyright (C) 2010 - 2024 Proxmox Server Solutions GmbH
 
 This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>

src/PVE/CLI/Makefile

@@ -1,4 +1,5 @@
-SOURCES=pvesm.pm
+SOURCES=pvesm.pm \
+		pvebcache.pm
 
 .PHONY: install
 install: ${SOURCES}

src/PVE/CLI/pvebcache.pm

@@ -0,0 +1,510 @@
+package PVE::CLI::pvebcache;
+
+use strict;
+use warnings;
+
+use PVE::Cluster;
+use PVE::APLInfo;
+use PVE::SafeSyslog;
+use PVE::Tools qw(extract_param file_read_firstline run_command) ;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::CLIHandler;
+use PVE::API2::Nodes;
+use PVE::Storage;
+use File::Basename;
+use Cwd 'realpath';
+use JSON;
+
+
+use base qw(PVE::CLIHandler);
+
+my $nodename = PVE::INotify::nodename();
+my $showbcache = "/usr/sbin/bcache-super-show";
+my $makebacahe = "/usr/sbin/make-bcache";
+my $LSBLK = "/bin/lsblk";
+
+
+sub setup_environment {
+    PVE::RPCEnvironment->setup_default_cli_env();
+}
+
+__PACKAGE__->register_method ({
+    name => 'index',
+    path => 'index',
+    method => 'GET',
+    description => "Get list of all templates on storage",
+    permissions => {
+	description => "Show all users the template which have permission on that storage."
+	},
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+		'type' => {
+			optional => 1,
+			type => 'string',
+			description => "Show bcache type",
+			enum => [qw(all cache backend)],
+			default => 'all',
+    	},
+	},
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+	my $type = $param->{type} // 'all';
+	#print Dumper($res);
+
+    my $devlist = PVE::Diskmanage::scan_bcache_device($type);
+
+	#print Dumper($devlist);
+    # foreach my $device (</sys/block/bcache*>) {
+	# 	#my $disk = basename($device);
+	# 	scan_bcache_device($devlist, $device, 0, 0);
+    # }
+
+	printf "%-10s %-10s %-20s %-20s  %-15s %-15s %-15s\n",
+	qw(name type backend-dev  cache-dev state size cachemode);
+	foreach my $rec ( @$devlist) {
+	    printf "%-10s %-10s %-20s %-20s %-15s %-15s %-15s \n", 
+			$rec->{name},
+			$rec->{type}, 
+			$rec->{'backend-dev'}, 
+			$rec->{'cache-dev'}, 
+	        $rec->{state},
+	        $rec->{size},
+			$rec->{cachemode} // 0
+		;
+	};
+
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'stop',
+    path => 'stop',
+    method => 'Post',
+    description => "Stop bcache",
+    permissions => {
+	description => "Stop bcache"
+	},
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+		dev => {
+            type => 'string',
+            title => 'bcache name'
+        }
+	},
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+
+	my $dev = $param->{dev};
+	my $sysdir = "/sys/block";
+
+	die "$dev is not a bcache dev format! \n" if $dev !~ m{bcache\d+$} ;
+
+	if ($dev =~ m{^/dev/bcache\d+$}) {
+		$dev = basename($dev);
+	}
+
+	die "Stop dev $dev failed!\n" if !PVE::SysFSTools::file_write("$sysdir/$dev/bcache/stop","1");
+
+    }});
+__PACKAGE__->register_method ({
+    name => 'register',
+    path => 'register',
+    method => 'Post',
+    description => "register a bcache",
+    permissions => {
+	description => "register a  bcache"
+	},
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+        dev => {
+            type => 'string',
+            title => 'dev name'
+        }
+	},
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+
+	my $dev =  PVE::Diskmanage::get_disk_name($param->{dev});
+	die "$dev has been a bcache dev!\n" if ( -d "/sys/block/$dev/bcache/");
+	return PVE::SysFSTools::file_write("/sys/fs/bcache/register","/dev/$dev");
+	
+    }});
+__PACKAGE__->register_method ({
+    name => 'create',
+    path => 'create',
+    method => 'Post',
+    description => "register a bcache",
+    permissions => {
+	description => "register a  bcache"
+	},
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+        backend => {
+            type => 'string',
+            title => 'backend dev name'
+        },
+		cache => {
+            type => 'string',
+            title => 'Cache dev name',
+			optional => 1,
+        },
+		blocksize => {
+            type => 'integer',
+            title => 'blocksize',
+			optional => 1,
+        },
+		writeback => {
+            type => 'boolean',
+            title => 'enable writeback',
+			default => 0,
+			optional => 1,
+        },
+		discard => {
+            type => 'boolean',
+            title => 'enable discard',
+			default => 1,
+			optional => 1,
+        },
+	},
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+
+	my $dev =  PVE::Diskmanage::get_disk_name($param->{backend});
+	my $cache = $param->{cache};
+	my $blocksize = $param->{blocksize};
+	my $writeback = $param->{writeback} // 1;
+	my $discard = $param->{discard} // 1;
+
+ 	die "backend $dev dev is not block device!" if !PVE::Diskmanage::verify_blockdev_path("/dev/$dev");
+	die "backend $dev dev has been a bcache device!\n"  if -d "/sys/block/$dev/bcache/"; 
+
+	my $cmd = ["$makebacahe","-B","/dev/$dev"];
+
+	if (defined($cache)){
+		die "$cache has been a cache dev,please create without cache and attach cache!\n" if check_bcache_cache_dev($cache);
+		$cache =  PVE::Diskmanage::get_disk_name($cache);
+		push @$cmd,"-C","/dev/$cache";
+	}
+
+	if (defined($blocksize)){
+		push @$cmd,"-w",$blocksize;
+	}
+
+	if (defined($writeback)){
+		push @$cmd,"--writeback";
+	}
+	if (defined($discard)){
+		push @$cmd,"--discard";
+	}
+
+	return	run_command($cmd , outfunc => sub {}, errfunc => sub {});
+
+    }});
+__PACKAGE__->register_method ({
+    name => 'detach',
+    path => 'detach',
+    method => 'POST',
+    description => "detach a cache dev",
+    permissions => {
+	description => "Show all users which have permission on that host."
+	},
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+		backend => {
+            type => 'string',
+			description => "backend dev",
+        },
+	}
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+
+	my $backenddev =  PVE::Diskmanage::get_bcache_backend_dev($param->{backend});
+	return PVE::SysFSTools::file_write("/sys/block/$backenddev/bcache/detach", "1");
+
+    }
+	});
+
+__PACKAGE__->register_method ({
+    name => 'attach',
+    path => 'attach',
+    method => 'POST',
+    description => "attach cache dev to backend dev",
+    permissions => {
+	description => "Show all users which have permission on that host."
+	},
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+		backend => {
+            type => 'string',
+			description => "backend dev",
+        },
+		cache => {
+            type => 'string',
+			description => "bcache dev",
+        },
+	}
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+
+	my $backenddev =  PVE::Diskmanage::get_bcache_backend_dev($param->{backend});
+	my $cachedev =  PVE::Diskmanage::get_bcache_cache_dev($param->{cache});
+
+	return PVE::SysFSTools::file_write("/sys/block/$backenddev/bcache/attach", $cachedev);
+
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'create_cache',
+    path => 'create_cache',
+    method => 'POST',
+    description => "ceate cache device",
+    permissions => {
+	description => "Show all users which have permission on that host."
+	},
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+		cache => {
+            type => 'string',
+			description => "cache dev",
+        },
+	}
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+	my $cachedev =  PVE::Diskmanage::get_disk_name($param->{cache});
+	my $cmd =[$makebacahe , "-C","/dev/$cachedev"];
+	return	run_command($cmd , outfunc => sub {}, errfunc => sub {});
+	
+
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'stop_cache',
+    path => 'stop_cache',
+    method => 'POST',
+    description => "stop cache device",
+    permissions => {
+	description => "Show all users which have permission on that host."
+	},
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+		cache => {
+            type => 'string',
+			description => "cache dev",
+        },
+	}
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+
+	my $cachedev = PVE::Diskmanage::get_bcache_cache_dev($param->{cache});
+	PVE::Diskmanage::check_bcache_cache_is_inuse($cachedev);
+	$cachedev =~ /^([a-zA-Z0-9_\-\.]+)$/ || die "Invalid cachedev format: $cachedev";
+	my $uuid = $1;
+	return PVE::SysFSTools::file_write("/sys/fs/bcache/$uuid/stop","1");
+    }});
+
+
+
+
+__PACKAGE__->register_method ({
+    name => 'set',
+    path => 'set',
+    method => 'POST',
+    description => "set backend device cache plicy",
+    permissions => {
+	description => "Show all users which have permission on that host."
+	},
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+		backend => {
+            type => 'string',
+			description => "backend dev",
+        },
+		cachemode => {
+            type => 'string',
+			description => "cache mode dev",
+			enum => [qw(writethrough writeback writearound none)],
+			optional => 1,
+        },
+		sequential => {
+            type => 'integer',
+			minimum => 0,
+			description => "Unit is in kb",
+			optional => 1,
+        },
+		'wb-percent' => {
+            type => 'integer',
+			minimum => 0,
+			maximum => 80,
+			description => "writeback_percent",
+			optional => 1,
+        },
+		'clear-stats' => {
+            type => 'boolean',
+			optional => 1,
+			default => 0,
+        },
+	}
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+
+	my $backenddev =  PVE::Diskmanage::get_bcache_backend_dev($param->{backend});
+	my $cachemode = $param->{cachemode};
+	my $sequential = $param->{sequential};
+	my $wb_percent = $param->{'wb-percent'};
+	my $clear = $param->{'clear-stats'};
+
+	if (!$clear && !$wb_percent && !$sequential && !$cachemode){
+		die "Need a param eg. --clear-stats 1 --wb-percent 20 --sequential 8192 --cachemode writeback\n";
+	}
+	my $path = "/sys/block/$backenddev/bcache";
+	sub write_to_file {
+		my ($file, $value) = @_;
+		eval {
+			if ($value){
+			my $old = file_read_firstline($file);
+			PVE::SysFSTools::file_write($file, $value) ;
+			my $new = file_read_firstline("$file");
+			my $name = basename($file);
+			print "$name: $old => $new \n";
+			}
+		};
+		warn $@ if $@;
+	}
+	write_to_file("$path/cache_mode", $cachemode);
+
+	write_to_file("$path/writeback_percent", $wb_percent);
+	if ($sequential){
+		$sequential = PVE::Tools::convert_size($sequential, 'kb' => 'b'); 
+		write_to_file("$path/sequential_cutoff", $sequential);
+	}
+
+	PVE::SysFSTools::file_write("$path/clear_stats", "1") if $clear;
+	return "ok\n";
+    }});
+
+
+our $cmddef = {
+	create => [  __PACKAGE__, 'create', [ 'backend' ] ,{ node => $nodename }],
+    stop => [  __PACKAGE__, 'stop', [ 'dev' ] ,{ node => $nodename }],
+	register => [  __PACKAGE__, 'register',[ 'dev' ] ,{ node => $nodename } ],
+    list => [  __PACKAGE__, 'index' , [],{ node => $nodename }],
+	start => { alias => 'register' },
+	cache => {
+		detach => [  __PACKAGE__, 'detach' , ['backend'], { node => $nodename }],
+		attach => [  __PACKAGE__, 'attach' , ['backend'], { node => $nodename }],
+		create => [  __PACKAGE__, 'create_cache' , ['cache'], { node => $nodename }],
+		stop => [  __PACKAGE__, 'stop_cache' , ['cache'], { node => $nodename }],
+		set => [  __PACKAGE__, 'set' , ['backend'], { node => $nodename }]
+	}
+
+};
+
+
+1;

src/PVE/Diskmanage.pm

@@ -5,7 +5,7 @@ use warnings;
 
 use PVE::ProcFSTools;
 use Data::Dumper;
-use Cwd qw(abs_path);
+use Cwd qw(abs_path realpath);
 use Fcntl ':mode';
 use File::Basename;
 use File::stat;
@@ -363,7 +363,9 @@ sub get_sysdir_size {
 sub get_sysdir_info {
     my ($sysdir) = @_;
 
-    return if ! -d "$sysdir/device";
+    if ($sysdir !~ /bcache\d+/ && ! -d "$sysdir/device") {
+        return;
+    }
 
     my $data = {};
 
@@ -374,7 +376,11 @@ sub get_sysdir_info {
 
     $data->{vendor} = file_read_firstline("$sysdir/device/vendor") || 'unknown';
     $data->{model} = file_read_firstline("$sysdir/device/model") || 'unknown';
-
+    if ($sysdir =~ /bcache\d+/){
+        $data->{vendor} = 'bcache';
+        $data->{model} = file_read_firstline("$sysdir/bcache/backing_dev_name") || 'unknown';
+        $data->{serial} = file_read_firstline("$sysdir/bcache/state") || 'unknown';
+    }
     return $data;
 }
 
@@ -522,7 +528,10 @@ sub get_disks {
 	# - cciss!cXnY  cciss devices
 	return if $dev !~ m/^(h|s|x?v)d[a-z]+$/ &&
 		  $dev !~ m/^nvme\d+n\d+$/ &&
-		  $dev !~ m/^cciss\!c\d+d\d+$/;
+		  $dev !~ m/^cciss\!c\d+d\d+$/ &&
+		  $dev !~ m/^mmcblk\d+n\d+$/ &&
+		  $dev !~ m/^nbd\d+n\d+$/ &&
+		  $dev !~ /bcache\d+/;
 
 	my $data = get_udev_info("/sys/block/$dev") // return;
 	my $devpath = $data->{devpath};
@@ -920,4 +929,182 @@ sub udevadm_trigger {
     warn $@ if $@;
 }
 
+
+sub scan_bcache_device {
+	my ($showtype) = @_;
+	my $ddd = []; 
+	my $res = get_lsblk_info();
+    foreach my $device (keys %$res) {
+        if ($res->{$device}{'fstype'} && $res->{$device}{'fstype'} eq 'bcache') {
+            my $d = {}; 
+			$device = basename($device);
+			my $path = get_bcache_dev_path($device);
+			my $state = "Stopped";
+			my $disktype = "unknown";
+			my $cachemode = "unknown";
+			my $backenddev = "unknown";
+			my $cache = "unknown";
+
+			if ( -d "/sys/block/$path/bcache/") {
+				$state = "Running";
+				$disktype = "backend";
+				#fix some bug
+				if ($device =~ m/^nvme/ ||
+				$device =~ m/^sd/ ||
+				$device =~ m/^xvd/ ||
+				$device =~ m/^mmcblk/ ||
+				$device =~ m/^nbd/) { 
+					if ( ! -d "/sys/block/$path/bcache/set") {
+						$device = basename(realpath("/sys/block/$path/bcache/dev")); 
+						$path = get_bcache_dev_path($device);
+					}
+				}
+            }
+
+			if ( -d "/sys/block/$path/bcache/set"){
+				$disktype = "cache";
+				$backenddev = "none";
+				$cachemode = "none";
+				$cache = "none";
+			}
+
+			if ($showtype ne 'all' && ($showtype ne $disktype)){
+				next;
+			}
+			
+			$d->{type} = $disktype;
+
+
+			#num3
+			$d->{state} = $state; 
+			if ( $disktype eq 'backend' && ( $showtype eq 'all' || $showtype eq $disktype  )){
+				if ( $state ne 'Stopped'){
+					$backenddev = file_read_firstline("/sys/block/$path/bcache/backing_dev_name");
+					$cachemode = file_read_firstline("/sys/block/$path/bcache/cache_mode");
+					if ( $cachemode && $cachemode =~ /\[(.*?)\]/) {
+						$cachemode = $1; 
+					}
+					if ( -d "/sys/block/$path/bcache/cache"){
+						$cache = basename(realpath("/sys/block/$path/bcache/cache/cache0/../"));
+					}
+				}else{
+					$backenddev = $device;
+				}
+			}
+            $d->{name} = $device;
+			$d->{'backend-dev'} = $backenddev;
+			$d->{cachemode} = $cachemode;
+			$d->{'cache-dev'} = $cache;
+
+            my $size = int(file_read_firstline("/sys/block/$path/size")) * 512;
+            $size = PVE::Tools::convert_size($size, 'b' => 'GB');
+			$d->{size} = $size . "GB"; 
+
+            push @$ddd, $d;
+        }
+    }
+
+	#print  Dumper($ddd);
+	@$ddd = sort { $a->{name} cmp $b->{name} } @$ddd;
+    return  $ddd;
+}
+
+
+sub get_devices_by_uuid {
+    my ($lsblk_info, $uuids, $res) = @_;
+
+    $res = {} if !defined($res);
+
+    foreach my $dev (sort keys %{$lsblk_info}) {
+	my $uuid = $lsblk_info->{$dev}->{uuid};
+	next if !defined($uuid) || !defined($uuids->{$uuid});
+	$res->{$dev} = $uuids->{$uuid};
+    }
+
+    return $res;
+}
+
+sub get_bcache_dev_path {
+	my ($dev) = @_;
+	my $diskname = $dev;
+	if ($dev =~ m/^(nvme\d+n\d+)p\d+$/ || # nvme分区，如nvme0n1p1
+                $dev =~ m/^([sv]d[a-z]+)\d+$/ ||  # sda1, vdb2等标准分区
+                $dev =~ m/^(xvd[a-z]+)\d+$/ ||    # xen虚拟分区
+                $dev =~ m/^(mmcblk\d+)p\d+$/ ||   # mmcblk分区
+                $dev =~ m/^(nbd\d+)p\d+$/) {      # nbd网络块设备分区
+				$diskname = $1;
+				$dev = "$diskname/$dev";
+            }
+	return $dev;
+}
+
+sub get_bcache_cache_dev {
+	my ($cachedev) = @_;
+	my $path = $cachedev;
+	if ($cachedev =~ m{^/dev/}) {
+		$cachedev = basename($cachedev);
+		$path = get_bcache_dev_path($cachedev);
+		die "$cachedev is not a bcache dev!\n" if (! -d "/sys/block/$path/bcache/set");
+		$cachedev = basename(realpath("/sys/block/$path/bcache/set"));
+	} elsif (is_uuid($cachedev)){
+		die "uuid $cachedev not a cache dev!\n" if (! -d "/sys/fs/bcache/$cachedev/");
+	} else  {
+		$path = get_bcache_dev_path($cachedev);
+		die "cache $cachedev dev is not a cache device!\n" if ! -d "/sys/block/$path/bcache/set";
+		$cachedev = basename(realpath("/sys/block/$path/bcache/set"));
+	}
+	return $cachedev;
+}
+
+sub check_bcache_cache_dev {
+	my ($cachedev) = @_;
+	if (is_uuid($cachedev)){
+		return 0 if (! -d "/sys/fs/bcache/$cachedev/");
+	}else{
+		my $path = get_bcache_dev_path($cachedev);
+		return 0 if (! -d "/sys/block/$path/bcache/set");
+	}
+	return 1;
+}
+
+sub is_uuid {
+    my ($uuid) = @_;
+    my $uuid_regex = qr/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+    return $uuid =~ $uuid_regex;
+}
+
+sub get_bcache_backend_dev {
+	my ($backenddev) = @_;
+	if ($backenddev =~ m{^/dev/}) {
+		$backenddev = basename($backenddev);
+	}
+	my $path = get_bcache_dev_path($backenddev);
+	die "backend $backenddev dev is not a bcache device!\n"  if ! -d "/sys/block/$path/bcache/"; 
+	return $backenddev;
+}
+
+sub check_bcache_cache_is_inuse {
+	my ($cache) = @_;
+	my @bdev = glob("/sys/fs/bcache/$cache/bdev*");
+	die "cache dev $cache is in use!\n" if scalar @bdev > 0;
+}
+
+sub get_disk_name {
+	my ($dev) = @_;
+	if ($dev =~ m{^/dev/}) {
+		$dev = basename($dev);
+	}
+	die "$dev is not a blockdev!\n" if !PVE::Diskmanage::verify_blockdev_path("/dev/$dev");
+	return $dev;
+}
+
+
+sub bcache_cache_uuid_to_dev {
+	my ($dev) = @_;
+	my $path = get_bcache_dev_path($dev);
+	return basename(realpath("/sys/block/$path/bcache/cache0/../"));
+}
+
+
+
 1;

src/PVE/Storage.pm

@@ -967,6 +967,24 @@ sub vdisk_clone {
     });
 }
 
+sub vdisk_clone_pxvirt{
+    my ($cfg, $volid, $vmid, $snap) = @_;
+
+    my ($storeid, $volname) = parse_volume_id($volid);
+
+    my $scfg = storage_config($cfg, $storeid);
+
+    my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
+
+    activate_storage($cfg, $storeid);
+
+    # lock shared storage
+    return $plugin->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub {
+        my $volname = $plugin->clone_image_pxvirt($scfg, $storeid, $volname, $vmid, $snap);
+        return "$storeid:$volname";
+    });
+}
+
 sub vdisk_create_base {
     my ($cfg, $volid) = @_;
 

src/PVE/Storage/LvmThinPlugin.pm

@@ -287,6 +287,31 @@ sub clone_image {
     return $name;
 }
 
+sub clone_image_pxvirt {
+    my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
+
+    my $vg = $scfg->{vgname};
+
+    my $lv;
+
+    if ($snap) {
+	$lv = "$vg/snap_${volname}_$snap";
+    } else {
+	my ($vtype, undef, undef, undef, undef, $isBase, $format) =
+	    $class->parse_volname($volname);
+
+
+	$lv = "$vg/$volname";
+    }
+
+    my $name = $class->find_free_diskname($storeid, $scfg, $vmid);
+
+    my $cmd = ['/sbin/lvcreate', '-n', $name, '-prw', '-kn', '-s', $lv];
+    run_command($cmd, errmsg => "clone image '$lv' error");
+
+    return $name;
+}
+
 sub create_base {
     my ($class, $storeid, $scfg, $volname) = @_;
 

src/PVE/Storage/Plugin.pm

@@ -897,6 +897,51 @@ sub clone_image {
     return $newvol;
 }
 
+
+sub clone_image_pxvirt {
+    my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
+
+    # this only works for file based storage types
+    die "storage definition has no path\n" if !$scfg->{path};
+
+    my ($vtype, $basename, $basevmid, undef, undef, $isBase, $format) =
+	$class->parse_volname($volname);
+
+	die "clone_image only works on ZFS/LVMthin/cephrbd\n" if !$isBase;
+
+    die "clone_image on wrong vtype '$vtype'\n" if $vtype ne 'images';
+
+    die "this storage type does not support clone_image on subvolumes\n" if $format eq 'subvol';
+
+    my $imagedir = $class->get_subdir($scfg, 'images');
+    $imagedir .= "/$vmid";
+
+    mkpath $imagedir;
+
+    my $name = $class->find_free_diskname($storeid, $scfg, $vmid, "qcow2", 1);
+
+    warn "clone $volname: $vtype, $name, $vmid to $name (base=../$basevmid/$basename)\n";
+
+    my $newvol = "$vmid/$name";
+
+    my $path = $class->filesystem_path($scfg, $newvol);
+
+    # Note: we use relative paths, so we need to call chdir before qemu-img
+    eval {
+	local $CWD = $imagedir;
+
+	my $cmd = ['/usr/bin/qemu-img', 'create', '-b', "../$basevmid/$basename",
+		   '-F', $format, '-f', 'qcow2', $path];
+
+	run_command($cmd);
+    };
+    my $err = $@;
+
+    die $err if $err;
+
+    return $newvol;
+}
+
 sub alloc_image {
     my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
 

src/PVE/Storage/RBDPlugin.pm

@@ -620,6 +620,43 @@ sub clone_image {
     return $newvol;
 }
 
+sub clone_image_pxvirt {
+    my ($class, $scfg, $storeid, $volname, $vmid, $snapname) = @_;
+
+    my $snap = '__base__';
+    $snap = $snapname if length $snapname;
+
+    my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
+        $class->parse_volname($volname);
+
+    my $name = $class->find_free_diskname($storeid, $scfg, $vmid);
+
+    warn "clone $volname: $basename snapname $snap to $name\n";
+
+    if (length($snapname)) {
+	my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $volname, $snapname);
+
+	if (!$protected) {
+	    my $cmd = $rbd_cmd->($scfg, $storeid, 'snap', 'protect', $volname, '--snap', $snapname);
+	    run_rbd_command($cmd, errmsg => "rbd protect $volname snap $snapname error");
+	}
+    }
+
+    my $newvol = "$basename/$name";
+    $newvol = $name if length($snapname);
+
+    my @options = (
+	get_rbd_path($scfg, $basename),
+	'--snap', $snap,
+    );
+    push @options, ('--data-pool', $scfg->{'data-pool'}) if $scfg->{'data-pool'};
+
+    my $cmd = $rbd_cmd->($scfg, $storeid, 'clone', @options, get_rbd_path($scfg, $name));
+    run_rbd_command($cmd, errmsg => "rbd clone '$basename' error");
+
+    return $newvol;
+}
+
 sub alloc_image {
     my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
 

src/PVE/Storage/ZFSPoolPlugin.pm

@@ -649,6 +649,27 @@ sub clone_image {
     return "$basename/$name";
 }
 
+sub clone_image_pxvirt {
+    my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
+
+    $snap ||= '__base__';
+
+    my ($vtype, $basename, $basevmid, undef, undef, $isBase, $format) =
+        $class->parse_volname($volname);
+
+    my $name = $class->find_free_diskname($storeid, $scfg, $vmid, $format);
+
+    if ($format eq 'subvol') {
+	my $size = $class->zfs_request($scfg, undef, 'list', '-Hp', '-o', 'refquota', "$scfg->{pool}/$basename");
+	chomp($size);
+	$class->zfs_request($scfg, undef, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name", '-o', "refquota=$size");
+    } else {
+	$class->zfs_request($scfg, undef, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name");
+    }
+
+    return "$name";
+}
+
 sub create_base {
     my ($class, $storeid, $scfg, $volname) = @_;
 
@@ -710,7 +731,7 @@ sub volume_has_feature {
 
     my $features = {
 	snapshot => { current => 1, snap => 1},
-	clone => { base => 1},
+	clone => { base => 1 , snap => 1}, # we add snap features for link clone  .
 	template => { current => 1},
 	copy => { base => 1, current => 1},
 	sparseinit => { base => 1, current => 1},

src/bin/Makefile

@@ -25,6 +25,8 @@ pvesm.zsh-completion:
 install: pvesm.1 pvesm.bash-completion pvesm.zsh-completion
 	install -d $(DESTDIR)$(SBINDIR)
 	install -m 0755 pvesm $(DESTDIR)$(SBINDIR)
+	install -m 0755 pvebcache $(DESTDIR)$(SBINDIR)
+	install -m 0755 pvebcache  $(DESTDIR)$(SBINDIR)
 	install -d $(DESTDIR)$(MAN1DIR)
 	install -m 0644 pvesm.1 $(DESTDIR)$(MAN1DIR)
 	gzip -9 -n $(DESTDIR)$(MAN1DIR)/pvesm.1

src/bin/pvebcache

@@ -0,0 +1,8 @@
+#!/usr/bin/perl -T
+
+use strict;
+use warnings;
+
+use PVE::CLI::pvebcache;
+
+PVE::CLI::pvebcache->run_cli_handler();

src/test/run_test_zfspoolplugin.pl

@@ -2180,7 +2180,7 @@ my $test3 = sub {
     }
 
     eval {
-	if (PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$vmdisk", 'test', 0)) {
+	if (!PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$vmdisk", 'test', 0)) {
 	    $count++;
 	    warn "Test3 g1 failed";
 	}
@@ -2191,7 +2191,7 @@ my $test3 = sub {
     }
 
     eval {
-	if (PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$vmbase", 'test', 0)) {
+	if (!PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$vmbase", 'test', 0)) {
 	    $count++;
 	    warn "Test3 h1 failed";
 	}
@@ -2202,7 +2202,7 @@ my $test3 = sub {
     }
 
     eval {
-	if (PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$vmbase\/$vmlinked", 'test', 0)) {
+	if (!PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$vmbase\/$vmlinked", 'test', 0)) {
 	    $count++;
 	    warn "Test3 h1 failed";
 	}
@@ -2213,7 +2213,7 @@ my $test3 = sub {
     }
 
     eval {
-	if (PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$ctdisk", 'test', 0)) {
+	if (!PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$ctdisk", 'test', 0)) {
 	    $count++;
 	    warn "Test3 i1 failed";
 	}
@@ -2224,7 +2224,7 @@ my $test3 = sub {
     }
 
     eval {
-	if (PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$ctbase", 'test', 0)) {
+	if (!PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$ctbase", 'test', 0)) {
 	    $count++;
 	    warn "Test3 j1 failed";
 	}
@@ -2235,7 +2235,7 @@ my $test3 = sub {
     }
 
     eval {
-	if (PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$ctbase\/$ctlinked", 'test', 0)) {
+	if (!PVE::Storage::volume_has_feature($cfg, 'clone', "$storagename:$ctbase\/$ctlinked", 'test', 0)) {
 	    $count++;
 	    warn "Test3 k1 failed";
 	}