2D Image Filters with OpenCV

In this blog post I'm giving you an example on how to do basic 2D image filtering using OpenCV and displaying the result instantly using JavaFX.




Image filtering means that you apply various transformations on a given image. Of course, image processing is math, and I'll assume since you stumbled by this blog you are familiar with the basic concepts of image processing - if not there are plenty of articles in the web which can give you a good overview. Wikipedia will always give you a broader view on the topic.

Like you've noticed in the past few posts on this blog I'm making myself familiar with the OpenCV library, and the best way to learn a new API is of course to read whats available and make your own experiments. In my case, I've made a JavaFX application which makes it easy to explore the different effects you can achieve by changing the kernel values and getting instant feedback.

Warning: This blog post is just about very basic filtering, and chances are high that some of the operations deriving from parameterizing the kernels have their own names and/or have more efficient implementations in OpenCV.

As a sidenote, if you don't already know Bret Victors talk on 'inventing on principle' you should definitely visit his web site. I've tried to make the program given below in a way that the user can experiment and maybe get new ideas about the whole problem, invent their own kernel for example. It's fun to change some values here and there ...

Like this you get an idea what's behind words like 'blurring' or 'sharpening', and find out that "finding edges" means nothing more than apply simple yet powerful mathematical operations on an 2D matrix.

I have provided some example kernels along with the application to give you some starting points - but feel free to explore the effects. Check out this page for an explanation of the used kernels in the application.

Another motivation for this blog post is to explore the feasibility of using Scala along with OpenCV and - I'm biased - I find it a very good match. Even more so if you use JavaFX to implement the GUI.

Below is the code, this is all you need for the video above. (yes - I've discovered iMovie! ;-) ).

	package net.ladstatt.apps
	import java.io.ByteArrayInputStream
	import java.io.File
	import scala.collection.mutable.ArrayBuffer
	import scala.util.Failure
	import scala.util.Success
	import scala.util.Try
	import org.opencv.core.CvType
	import org.opencv.core.Mat
	import org.opencv.core.MatOfByte
	import org.opencv.highgui.Highgui
	import org.opencv.imgproc.Imgproc
	import javafx.application.Application
	import javafx.beans.value.ChangeListener
	import javafx.beans.value.ObservableValue
	import javafx.collections.ListChangeListener
	import javafx.event.Event
	import javafx.event.EventHandler
	import javafx.geometry.Pos
	import javafx.scene.Scene
	import javafx.scene.control.ChoiceBox
	import javafx.scene.control.ListCell
	import javafx.scene.control.ListView
	import javafx.scene.control.TextField
	import javafx.scene.image.Image
	import javafx.scene.image.ImageView
	import javafx.scene.layout.GridPane
	import javafx.scene.layout.HBox
	import javafx.scene.layout.VBox
	import javafx.stage.Stage
	import javafx.util.Callback
	/**
	* For a discussion of the concepts of this application see http://ladstatt.blogspot.com/
	*/
	trait Utils {
	val runOnMac =
	{
	System.getProperty("os.name").toLowerCase match {
	case "mac os x" => true
	case _ => false
	}
	}
	/**
	* function to measure execution time of first function, optionally executing a display function,
	* returning the time in milliseconds
	*/
	def time[A](a: => A, display: Long => Unit = s => ()): A = {
	val now = System.nanoTime
	val result = a
	val micros = (System.nanoTime - now) / 1000
	display(micros)
	result
	}
	}
	trait OpenCVUtils extends Utils {
	def loadNativeLibs() = {
	val nativeLibName = if (runOnMac) "/opt/local/share/OpenCV/java/libopencv_java244.dylib" else "c:/openCV/build/java/x64/opencv_java244.dll"
	System.load(new File(nativeLibName).getAbsolutePath())
	}
	def filter2D(kernel: Mat)(input: Mat): Mat = {
	val out = new Mat
	Imgproc.filter2D(input, out, -1, kernel)
	out
	}
	def toImage(mat: Mat): Try[Image] =
	try {
	val memory = new MatOfByte
	Highgui.imencode(".png", mat, memory)
	Success(new Image(new ByteArrayInputStream(memory.toArray())))
	} catch {
	case e: Throwable => Failure(e)
	}
	}
	trait JfxUtils {
	def mkChangeListener[T](onChangeAction: (ObservableValue[_ <: T], T, T) => Unit): ChangeListener[T] = {
	new ChangeListener[T]() {
	override def changed(observable: ObservableValue[_ <: T], oldValue: T, newValue: T) = {
	onChangeAction(observable, oldValue, newValue)
	}
	}
	}
	def mkListChangeListener[E](onChangedAction: ListChangeListener.Change[_ <: E] => Unit) = new ListChangeListener[E] {
	def onChanged(changeItem: ListChangeListener.Change[_ <: E]): Unit = {
	onChangedAction(changeItem)
	}
	}
	def mkCellFactoryCallback[T](listCellGenerator: ListView[T] => ListCell[T]) = new Callback[ListView[T], ListCell[T]]() {
	override def call(list: ListView[T]): ListCell[T] = listCellGenerator(list)
	}
	def mkEventHandler[E <: Event](f: E => Unit) = new EventHandler[E] { def handle(e: E) = f(e) }
	}
	/**
	* a variable sized gridpane, constrained by size
	*/
	class KernelInputArray(size: Int, kernelData: ArrayBuffer[Float], applyKernel: => Mat => Unit) extends GridPane with JfxUtils with OpenCVUtils {
	def mkKernel = {
	val kernel = new Mat(size, size, CvType.CV_32FC1)
	kernel.put(0, 0, kernelData.toArray)
	kernel
	}
	for {
	row <- 0 until size
	col <- 0 until size
	} yield {
	val textField = {
	val tf = new TextField
	tf.setPrefWidth(50)
	tf.setText(kernelData(row * size + col).toString)
	tf.textProperty().addListener(
	mkChangeListener[String](
	(obVal, oldVal, newVal) => {
	try {
	kernelData(row * size + col) = tf.getText.toFloat
	applyKernel(mkKernel)
	} catch {
	case e => // no float, ignore
	}
	}))
	tf
	}
	GridPane.setRowIndex(textField, row)
	GridPane.setColumnIndex(textField, col)
	getChildren.add(textField)
	}
	applyKernel(mkKernel)
	}
	object OpenCVFilter2D {
	def main(args: Array[String]): Unit = {
	Application.launch(classOf[OpenCVFilter2D], args: _*)
	}
	}
	class OpenCVFilter2D extends Application with JfxUtils with OpenCVUtils {
	override def init(): Unit = loadNativeLibs // important to have this statement on the "right" thread
	def readImage(file: File): Mat = Highgui.imread(file.getAbsolutePath()) // , 0)
	/**
	* a map of predefined kernels which can serve as starting points for your experiments
	*/
	def kernels: Map[String, (Int, ArrayBuffer[Float], Float, Float)] = Map(
	"unit" -> (3, ArrayBuffer[Float](
	0, 0, 0,
	0, 1, 0,
	0, 0, 0), 1, 0),
	"blur" -> (3, ArrayBuffer[Float](
	0, 0.2f, 0,
	0.2f, 0.2f, 0.2f,
	0, 0.2f, 0), 1, 0),
	"moreblur" -> (5, ArrayBuffer[Float](
	0, 0, 1, 0, 0,
	0, 1, 1, 1, 0,
	1, 1, 1, 1, 1,
	0, 1, 1, 1, 0,
	0, 0, 1, 0, 0), 1f / 13, 0),
	"motionblur" -> (9, ArrayBuffer[Float](
	1, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 1, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 1, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 1, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 1, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 1, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 1, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 1, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 1), 1f / 9, 0),
	"horizontaledges" -> (5, ArrayBuffer[Float](
	0, 0, 0, 0, 0,
	0, 0, 0, 0, 0,
	-1, -1, 2, 0, 0,
	0, 0, 0, 0, 0,
	0, 0, 0, 0, 0), 1, 0),
	"verticaledges" -> (5, ArrayBuffer[Float](
	0, 0, -1, 0, 0,
	0, 0, -1, 0, 0,
	0, 0, 4, 0, 0,
	0, 0, -1, 0, 0,
	0, 0, -1, 0, 0), 1, 0),
	"diagonaledges" -> (5, ArrayBuffer[Float](
	-1, 0, 0, 0, 0,
	0, -2, 0, 0, 0,
	0, 0, 6, 0, 0,
	0, 0, 0, -2, 0,
	0, 0, 0, 0, -1), 1, 0),
	"alledges" -> (3, ArrayBuffer[Float](
	-1, -1, -1,
	-1, 8, -1,
	-1, -1, -1), 1, 0),
	"sharpen" -> (3, ArrayBuffer[Float](
	-1, -1, -1,
	-1, 9, -1,
	-1, -1, -1), 1, 0),
	"subtlesharpen" -> (5, ArrayBuffer[Float](
	-1, -1, -1, -1, -1,
	-1, 2, 2, 2, -1,
	-1, 2, 8, 2, -1,
	-1, 2, 2, 2, -1,
	-1, -1, -1, -1, -1), 1f / 8, 0),
	"excesssharpen" -> (3, ArrayBuffer[Float](
	1, 1, 1,
	1, -7, 1,
	1, 1, 1), 1, 0),
	"emboss" -> (3, ArrayBuffer[Float](
	-1, -1, 0,
	-1, 0, 1,
	0, 1, 1), 1f, 0.1f),
	"emboss2" -> (5, ArrayBuffer[Float](-1, -1, -1, -1, 0,
	-1, -1, -1, 0, 1,
	-1, -1, 0, 1, 1,
	-1, 0, 1, 1, 1,
	0, 1, 1, 1, 1), 1, 0))
	def mkKernelInputArray(initialKernelName: String, mutateFn: => Mat => Unit): KernelInputArray = {
	val (size, kernelData, factor, bias) = kernels(initialKernelName)
	val initalKernel = kernelData.map(_ * factor + bias)
	new KernelInputArray(size, initalKernel, mutateFn)
	}
	override def start(stage: Stage): Unit = {
	stage.setTitle("2D Image Filters with OpenCV and JavaFX")
	val canvas = new HBox
	canvas.setAlignment(Pos.CENTER)
	val choiceBox = new ChoiceBox[String]
	choiceBox.getItems.addAll(kernels.keySet.toSeq.sortWith(_ < _): _*)
	choiceBox.setValue("unit")
	val input = readImage(new File("src/main/resources/turbine.png"))
	toImage(input) match {
	case Failure(e) => println(e.getMessage())
	case Success(inputImage) => {
	def mutateOutputImage(outputView: ImageView)(kernel: Mat): Unit = {
	toImage(filter2D(kernel)(input)) match {
	case Failure(e) =>
	case Success(mutatedImage) => outputView.setImage(mutatedImage)
	}
	}
	val originalView = new ImageView(inputImage)
	val outputView = new ImageView(inputImage) // show input image initially
	choiceBox.valueProperty().addListener(mkChangeListener[String](
	(obVal, oldVal, newVal) => {
	val cell = mkKernelInputArray(newVal, mutateOutputImage(outputView))
	canvas.getChildren().clear()
	val kernelAndChoiceBox = new VBox
	kernelAndChoiceBox.setAlignment(Pos.CENTER)
	kernelAndChoiceBox.getChildren.addAll(choiceBox, cell)
	canvas.getChildren().addAll(originalView, kernelAndChoiceBox, outputView)
	}))
	val kernelAndChoiceBox = new VBox
	kernelAndChoiceBox.setAlignment(Pos.CENTER)
	kernelAndChoiceBox.getChildren.addAll(choiceBox, mkKernelInputArray("unit", mutateOutputImage(outputView)))
	canvas.getChildren().addAll(originalView, kernelAndChoiceBox, outputView)
	}
	}
	val scene = new Scene(canvas)
	stage.setScene(scene)
	stage.show
	}
	}

view raw OpenCVFilter2D.scala hosted with ❤ by GitHub