fpga-admissionwebhook: add initial implementation

2025-06-03 03:59:37 +00:00 · 2018-06-05 17:14:00 +03:00 · 2018-06-05 17:14:00 +03:00 · 562f8fe722
commit 562f8fe722
parent 00b0cb92ea
5 changed files with 719 additions and 0 deletions
--- a/README.md
+++ b/README.md
@ -7,6 +7,7 @@
 - [About](#about)
 - [GPU device plugin](cmd/gpu_plugin/README.md)
 - [FPGA device plugin](cmd/fpga_plugin/README.md)
+- [FPGA admission controller webhook](cmd/fpga_admissionwebhook/README.md)

 ## About

--- a/build/docker/intel-fpga-admissionwebhook.Dockerfile
+++ b/build/docker/intel-fpga-admissionwebhook.Dockerfile
@ -0,0 +1,10 @@
+FROM golang:1.10-alpine as builder
+ARG DIR=/go/src/github.com/intel/intel-device-plugins-for-kubernetes
+WORKDIR $DIR
+COPY . .
+RUN cd cmd/fpga_admissionwebhook; go install
+RUN chmod a+x /go/bin/fpga_admissionwebhook
+
+FROM alpine
+COPY --from=builder /go/bin/fpga_admissionwebhook /usr/bin/intel_fpga_admissionwebhook
+CMD ["/usr/bin/intel_fpga_admissionwebhook"]
--- a/cmd/fpga_admissionwebhook/README.md
+++ b/cmd/fpga_admissionwebhook/README.md
@ -0,0 +1,240 @@
+## Build and install Intel FPGA webhook for admission controller
+
+### Get source code
+
+    $ mkdir -p $GOPATH/src/github.com/intel/
+    $ cd $GOPATH/src/github.com/intel/
+    $ git clone https://github.com/intel/intel-device-plugins-for-kubernetes.git
+
+### Build a Docker image with the webhook
+
+    $ cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
+    $ make webhook-container
+    $ docker images
+    REPOSITORY                    TAG                                        IMAGE ID            CREATED          SIZE
+    intel-fpga-admissionwebhook   878381826cdef0b112234c296d4e13d3266455ae   0fbbf9dfae95        0 sec ago        24.7MB
+    ...
+
+### Create secret including signed key/cert pair for the webhook
+
+Use the following script taken from [this article](https://medium.com/ibm-cloud/diving-into-kubernetes-mutatingadmissionwebhook-6ef3c5695f74):
+
+```bash
+#!/bin/bash
+
+while [[ $# -gt 0 ]]; do
+    case ${1} in
+        --service)
+            service="$2"
+            shift
+            ;;
+        --secret)
+            secret="$2"
+            shift
+            ;;
+        --namespace)
+            namespace="$2"
+            shift
+            ;;
+	--kubectl)
+	    kubectl="$2"
+            shift
+            ;;
+    esac
+    shift
+done
+
+[ -z ${service} ] && service=intel-fpga-webhook-svc
+[ -z ${secret} ] && secret=intel-fpga-webhook-certs
+[ -z ${namespace} ] && namespace=default
+[ -z ${kubectl} ] && kubectl=kubectl
+
+csrName=${service}.${namespace}
+tmpdir=$(mktemp -d)
+echo "creating certs in tmpdir ${tmpdir} "
+
+cat <<EOF >> ${tmpdir}/csr.conf
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = ${service}
+DNS.2 = ${service}.${namespace}
+DNS.3 = ${service}.${namespace}.svc
+EOF
+
+openssl genrsa -out ${tmpdir}/server-key.pem 2048
+openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf
+
+# clean-up any previously created CSR for our service. Ignore errors if not present.
+${kubectl} delete csr ${csrName} 2>/dev/null || true
+
+# create  server cert/key CSR and  send to k8s API
+cat <<EOF | ${kubectl} create -f -
+apiVersion: certificates.k8s.io/v1beta1
+kind: CertificateSigningRequest
+metadata:
+  name: ${csrName}
+spec:
+  groups:
+  - system:authenticated
+  request: $(cat ${tmpdir}/server.csr | base64 -w 0)
+  usages:
+  - digital signature
+  - key encipherment
+  - server auth
+EOF
+
+# verify CSR has been created
+while true; do
+    ${kubectl} get csr ${csrName}
+    if [ "$?" -eq 0 ]; then
+        break
+    fi
+done
+
+# approve and fetch the signed certificate
+${kubectl} certificate approve ${csrName}
+
+# verify certificate has been signed
+for x in $(seq 10); do
+    serverCert=$(${kubectl} get csr ${csrName} -o jsonpath='{.status.certificate}')
+    if [[ ${serverCert} != '' ]]; then
+        break
+    fi
+    sleep 1
+done
+
+if [[ ${serverCert} == '' ]]; then
+    echo "ERROR: After approving csr ${csrName}, the signed certificate did not appear on the resource. Giving up after 10 attempts." >&2
+    exit 1
+fi
+
+echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem
+
+
+# create the secret with CA cert and server cert/key
+${kubectl} create secret generic ${secret} \
+        --from-file=key.pem=${tmpdir}/server-key.pem \
+        --from-file=cert.pem=${tmpdir}/server-cert.pem \
+        --dry-run -o yaml |
+    ${kubectl} -n ${namespace} apply -f -
+```
+
+You should see something like
+
+    $ ./scripts/webhook-create-signed-cert.sh
+    creating certs in tmpdir /tmp/tmp.9sgk16v5Y2 
+    Generating RSA private key, 2048 bit long modulus
+    .........................+++
+    ....+++
+    e is 65537 (0x010001)
+    certificatesigningrequest "intel-fpga-webhook-svc.default" created
+    NAME                             AGE       REQUESTOR      CONDITION
+    intel-fpga-webhook-svc.default   0s        system:admin   Pending
+    certificatesigningrequest "intel-fpga-webhook-svc.default" approved
+    secret "intel-fpga-webhook-certs" created
+
+### Deploy webhook service
+
+Using the following specs:
+
+```yaml
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+      name: intel-fpga-webhook-deployment
+      labels:
+        app: intel-fpga-webhook
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: intel-fpga-webhook
+    spec:
+      containers:
+        - name: fpga-mutator
+          image: intel-fpga-admissionwebhook:878381826cdef0b112234c296d4e13d3266455ae
+          imagePullPolicy: IfNotPresent
+          command:
+            - /usr/bin/intel_fpga_admissionwebhook
+          args:
+            - -tls-cert-file=/etc/webhook/certs/cert.pem
+            - -tls-private-key-file=/etc/webhook/certs/key.pem
+            - -alsologtostderr
+            - -v=2
+            - 2>&1
+          volumeMounts:
+            - name: webhook-certs
+              mountPath: /etc/webhook/certs
+              readOnly: true
+      volumes:
+        - name: webhook-certs
+          secret:
+            secretName: intel-fpga-webhook-certs
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: intel-fpga-webhook-svc
+  labels:
+    app: intel-fpga-webhook
+spec:
+  ports:
+  - port: 443
+    targetPort: 443
+  selector:
+    app: intel-fpga-webhook
+```
+
+create `intel-fpga-webhook-svc` service:
+
+    $ kubectl create -f </path/to/deployment.yaml>
+    deployment.extensions/intel-fpga-webhook-deployment created
+    service/intel-fpga-webhook-svc created
+
+### Configure webhook admission controller on the fly
+
+With this spec as a template
+```yaml
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: fpga-mutator-webhook-cfg
+  labels:
+    app: intel-fpga-webhook
+webhooks:
+- name: fpga.mutator.webhooks.intel.com
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - pods
+  clientConfig:
+    service:
+      path: "/pods"
+      namespace: default
+      name: intel-fpga-webhook-svc
+    caBundle: {CA_BUNDLE}
+```
+
+register the webhook
+
+    $ cat </path/to/fpga-pod-mutating-webhook.yaml> | sed -e "s/{CA_BUNDLE}/$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 -w 0)/g" | kubectl create -f -
+
+Please note that the placeholder `{CA_BUNDLE}` should be replaced with the
+certificate which is used for signing certificate requests in your cluster,
+the one passed in the option `--cluster-signing-cert-file` to
+`kube-controller-manager`. Depending on how your cluster is configured it may
+differ from what is stored in the configmap `extension-apiserver-authentication`.
--- a/cmd/fpga_admissionwebhook/fpga_admissionwebhook.go
+++ b/cmd/fpga_admissionwebhook/fpga_admissionwebhook.go
@ -0,0 +1,257 @@
+// Copyright 2018 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/golang/glog"
+
+	"k8s.io/api/admission/v1beta1"
+	admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+const (
+	resourceReplaceOp = `{
+                "op": "remove",
+                "path": "/spec/containers/%d/resources/%s/%s"
+        }, {
+                "op": "add",
+                "path": "/spec/containers/%d/resources/%s/%s",
+                "value": %s
+        }`
+)
+
+var (
+	scheme         = runtime.NewScheme()
+	codecs         = serializer.NewCodecFactory(scheme)
+	rfc6901Escaper = strings.NewReplacer("~", "~0", "/", "~1")
+)
+
+func init() {
+	addToScheme(scheme)
+}
+
+func addToScheme(scheme *runtime.Scheme) {
+	corev1.AddToScheme(scheme)
+	admissionregistrationv1beta1.AddToScheme(scheme)
+}
+
+// TODO: get rid of hardcoded translations of FPGA resource names to region interface IDs
+func translateFpgaResourceName(oldname corev1.ResourceName) string {
+	switch string(oldname) {
+	case "intel.com/fpga-arria10":
+		return rfc6901Escaper.Replace("intel.com/fpga-region-ce48969398f05f33946d560708be108a")
+	case "intel.com/fpga-arria10-nlb0":
+		return rfc6901Escaper.Replace("intel.com/fpga-af-d8424dc4a4a3c413f89e433683f9040b")
+	case "intel.com/fpga-arria10-nlb3":
+		return rfc6901Escaper.Replace("intel.com/fpga-af-f7df405cbd7acf7222f144b0b93acd18")
+	}
+
+	return ""
+}
+
+func getTLSConfig(certFile string, keyFile string) *tls.Config {
+	sCert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		glog.Fatal(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{sCert},
+	}
+}
+
+func mutatePods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
+	var ops []string
+
+	glog.V(2).Info("mutating pods")
+
+	podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
+	if ar.Request.Resource != podResource {
+		glog.Errorf("expect resource to be %s", podResource)
+		return nil
+	}
+
+	raw := ar.Request.Object.Raw
+	pod := corev1.Pod{}
+	deserializer := codecs.UniversalDeserializer()
+	if _, _, err := deserializer.Decode(raw, nil, &pod); err != nil {
+		glog.Error(err)
+		return toAdmissionResponse(err)
+	}
+	reviewResponse := v1beta1.AdmissionResponse{}
+	reviewResponse.Allowed = true
+
+	for containerIdx, container := range pod.Spec.Containers {
+		for resourceName, resourceQuantity := range container.Resources.Limits {
+			newName := translateFpgaResourceName(resourceName)
+			if len(newName) > 0 {
+				op := fmt.Sprintf(resourceReplaceOp, containerIdx,
+					"limits", rfc6901Escaper.Replace(string(resourceName)),
+					containerIdx, "limits", newName, resourceQuantity.String())
+				ops = append(ops, op)
+			}
+		}
+		for resourceName, resourceQuantity := range container.Resources.Requests {
+			newName := translateFpgaResourceName(resourceName)
+			if len(newName) > 0 {
+				op := fmt.Sprintf(resourceReplaceOp, containerIdx,
+					"requests", rfc6901Escaper.Replace(string(resourceName)),
+					containerIdx, "requests", newName, resourceQuantity.String())
+				ops = append(ops, op)
+			}
+		}
+	}
+
+	if len(ops) > 0 {
+		reviewResponse.Patch = []byte("[ " + strings.Join(ops, ",") + " ]")
+		pt := v1beta1.PatchTypeJSONPatch
+		reviewResponse.PatchType = &pt
+	}
+
+	return &reviewResponse
+}
+
+type admitFunc func(v1beta1.AdmissionReview) *v1beta1.AdmissionResponse
+
+func toAdmissionResponse(err error) *v1beta1.AdmissionResponse {
+	return &v1beta1.AdmissionResponse{
+		Result: &metav1.Status{
+			Message: err.Error(),
+		},
+	}
+}
+
+func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) {
+	var body []byte
+	var reviewResponse *v1beta1.AdmissionResponse
+	var reqUID types.UID
+
+	if r.Body != nil {
+		if data, err := ioutil.ReadAll(r.Body); err == nil {
+			body = data
+		}
+	}
+
+	if len(body) == 0 {
+		glog.Error("No body in request")
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	// verify the content type is accurate
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/json" {
+		glog.Errorf("contentType=%s, expect application/json", contentType)
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	glog.V(2).Info(fmt.Sprintf("handling request: %v", body))
+	ar := v1beta1.AdmissionReview{}
+	deserializer := codecs.UniversalDeserializer()
+	if _, _, err := deserializer.Decode(body, nil, &ar); err != nil {
+		glog.Error(err)
+		reviewResponse = toAdmissionResponse(err)
+	} else {
+		if ar.Request == nil {
+			err = errors.New("Request is empty")
+			reviewResponse = toAdmissionResponse(err)
+		} else {
+			reqUID = ar.Request.UID
+			reviewResponse = admit(ar)
+		}
+	}
+	glog.V(2).Info(fmt.Sprintf("sending response: %v", reviewResponse))
+
+	response := v1beta1.AdmissionReview{}
+	if reviewResponse != nil {
+		response.Response = reviewResponse
+		response.Response.UID = reqUID
+	}
+
+	// reset the Object and OldObject, they are not needed in a response.
+	if ar.Request != nil {
+		ar.Request.Object = runtime.RawExtension{}
+		ar.Request.OldObject = runtime.RawExtension{}
+	}
+
+	resp, err := json.Marshal(response)
+	if err != nil {
+		glog.Error(err)
+	}
+	if _, err := w.Write(resp); err != nil {
+		glog.Error(err)
+	}
+}
+
+func servePods(w http.ResponseWriter, r *http.Request) {
+	serve(w, r, mutatePods)
+}
+
+func main() {
+	var certFile string
+	var keyFile string
+
+	flag.StringVar(&certFile, "tls-cert-file", certFile,
+		"File containing the x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert).")
+	flag.StringVar(&keyFile, "tls-private-key-file", keyFile, "File containing the x509 private key matching --tls-cert-file.")
+
+	flag.Parse()
+
+	if certFile == "" {
+		glog.Error("TLS certificate file is not set")
+		os.Exit(1)
+	}
+
+	if keyFile == "" {
+		glog.Error("TLS private key is not set")
+		os.Exit(1)
+	}
+
+	if _, err := os.Stat(certFile); err != nil {
+		glog.Error("TLS certificate not found")
+		os.Exit(1)
+	}
+
+	if _, err := os.Stat(keyFile); err != nil {
+		glog.Error("TLS private key not found")
+		os.Exit(1)
+	}
+
+	http.HandleFunc("/pods", servePods)
+
+	glog.V(2).Info("Webhook started")
+
+	server := &http.Server{
+		Addr:      ":443",
+		TLSConfig: getTLSConfig(certFile, keyFile),
+	}
+
+	glog.Fatal(server.ListenAndServeTLS("", ""))
+}
--- a/cmd/fpga_admissionwebhook/fpga_admissionwebhook_test.go
+++ b/cmd/fpga_admissionwebhook/fpga_admissionwebhook_test.go
@ -0,0 +1,211 @@
+// Copyright 2018 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"k8s.io/api/admission/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func fakeMutatePods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
+	reviewResponse := v1beta1.AdmissionResponse{}
+	return &reviewResponse
+}
+
+func TestServe(t *testing.T) {
+	ar1, err := json.Marshal(&v1beta1.AdmissionReview{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ar2, err := json.Marshal(&v1beta1.AdmissionReview{
+		Request: &v1beta1.AdmissionRequest{},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tcases := []struct {
+		header         http.Header
+		body           io.Reader
+		expectedStatus int
+	}{
+		{
+			expectedStatus: http.StatusBadRequest,
+		},
+		{
+			body:           strings.NewReader("hello world"),
+			expectedStatus: http.StatusBadRequest,
+		},
+		{
+			header: http.Header{
+				"Content-Type": []string{"application/json"},
+			},
+			expectedStatus: http.StatusBadRequest,
+		},
+		{
+			body: strings.NewReader("hello world"),
+			header: http.Header{
+				"Content-Type": []string{"application/json"},
+			},
+			expectedStatus: http.StatusOK,
+		},
+		{
+			body: bytes.NewReader(ar1),
+			header: http.Header{
+				"Content-Type": []string{"application/json"},
+			},
+			expectedStatus: http.StatusOK,
+		},
+		{
+			body: bytes.NewReader(ar2),
+			header: http.Header{
+				"Content-Type": []string{"application/json"},
+			},
+			expectedStatus: http.StatusOK,
+		},
+	}
+
+	for _, tcase := range tcases {
+		req, err := http.NewRequest("POST", "/pods", tcase.body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		req.Header = tcase.header
+
+		rr := httptest.NewRecorder()
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { serve(w, r, fakeMutatePods) })
+
+		handler.ServeHTTP(rr, req)
+
+		if status := rr.Code; status != tcase.expectedStatus {
+			t.Errorf("handler returned wrong status code: got %v want %v",
+				status, tcase.expectedStatus)
+		}
+
+		if tcase.expectedStatus == http.StatusOK {
+			var ar v1beta1.AdmissionReview
+			err = json.Unmarshal(rr.Body.Bytes(), &ar)
+			if err != nil {
+				t.Error(err)
+			}
+		}
+	}
+}
+
+func TestMutatePods(t *testing.T) {
+	pod := corev1.Pod{
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:  "test-container",
+					Image: "test-image",
+					Resources: corev1.ResourceRequirements{
+						Limits: corev1.ResourceList{
+							"cpu": resource.MustParse("1"),
+							"intel.com/fpga-arria10": resource.MustParse("1"),
+						},
+						Requests: corev1.ResourceList{
+							"cpu": resource.MustParse("1"),
+							"intel.com/fpga-arria10": resource.MustParse("1"),
+						},
+					},
+				},
+			},
+		},
+	}
+	podRaw, err := json.Marshal(pod)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tcases := []struct {
+		name             string
+		ar               v1beta1.AdmissionReview
+		expectedResponse bool
+		expectedPatchOps int
+	}{
+		{
+			name: "empty admission request",
+			ar: v1beta1.AdmissionReview{
+				Request: &v1beta1.AdmissionRequest{},
+			},
+		},
+		{
+			name: "admission request without object",
+			ar: v1beta1.AdmissionReview{
+				Request: &v1beta1.AdmissionRequest{
+					Resource: metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"},
+				},
+			},
+			expectedResponse: true,
+		},
+		{
+			name: "admission request with corrupted object",
+			ar: v1beta1.AdmissionReview{
+				Request: &v1beta1.AdmissionRequest{
+					Resource: metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"},
+					Object: runtime.RawExtension{
+						Raw: []byte(`{"corrupted json":}`),
+					},
+				},
+			},
+			expectedResponse: true,
+		},
+		{
+			name: "non-empty admission request",
+			ar: v1beta1.AdmissionReview{
+				Request: &v1beta1.AdmissionRequest{
+					Resource: metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"},
+					Object: runtime.RawExtension{
+						Raw: podRaw,
+					},
+				},
+			},
+			expectedResponse: true,
+			expectedPatchOps: 4,
+		},
+	}
+
+	for _, tcase := range tcases {
+		resp := mutatePods(tcase.ar)
+
+		if !tcase.expectedResponse && resp != nil {
+			t.Errorf("Test case '%s': got unexpected response", tcase.name)
+		} else if tcase.expectedResponse && resp == nil {
+			t.Errorf("Test case '%s': got no response", tcase.name)
+		} else if tcase.expectedResponse && tcase.expectedPatchOps > 0 {
+			var ops interface{}
+
+			err := json.Unmarshal(resp.Patch, &ops)
+			if err != nil {
+				t.Errorf("Test case '%s': got unparsable patch '%s'", tcase.name, resp.Patch)
+			} else if len(ops.([]interface{})) != tcase.expectedPatchOps {
+				t.Errorf("Test case '%s': got wrong number of operations in the patch. Expected %d, but got %d",
+					tcase.name, tcase.expectedPatchOps, len(ops.([]interface{})))
+			}
+		}
+	}
+}