Digital Signal Processing #1

The following example uses sequence and array operators to implement a simple digital filter. The example builds on the previous WAV I/O example. A digital filter frequently needs multiply-add operators, these may be implemented using Array.map2

> let fx = [|for x in {1..5} do yield sin((float)x)|];; 

val fx : float [] =
  [|0.8414709848; 0.9092974268; 0.1411200081; -0.7568024953; -0.9589242747|]

> let filter = [|1.0; -1.0; 0.0; 0.0; 0.0|];;

val filter : float [] = [|1.0; -1.0; 0.0; 0.0; 0.0|]

> Array.map2 (fun i j -> i*j) fx filter;;
val it : float [] = [|0.8414709848; -0.9092974268; 0.0; 0.0; 0.0|]

Full source:

open System
open System.IO
open System.Text

// https://ccrma.stanford.edu/courses/422/projects/WaveFormat/

type WavAudioFormat = WavPCM
exception WavUnknownFormat
exception WavUnknownNumChannels
exception WavUnknownBitsPerSample
exception WavUnknownFilter

let Sample16BitLE2Float i = float(BitConverter.ToInt16((Seq.toArray i), 0))/32768.0
let Float2Sample16BitLE s = BitConverter.GetBytes(int16(s*32768.0))

let breakBy windowSize (source: seq<_>) =
    seq { let arr = Array.create windowSize (Seq.nth 0 source)
          let i = ref 0
          use e = source.GetEnumerator()
          while e.MoveNext() do
              arr.[!i] <- e.Current
              i := !i + 1
              if !i = windowSize then
                yield Array.init windowSize (fun j -> arr.[j])
                i := 0
        }

type WavFile() =
  let mutable AudioFormat:WavAudioFormat = WavPCM
  let mutable NumChannels:int = 0
  let mutable SampleRate:int = 0
  let mutable BitsPerSample:int = 0
  let mutable Samples:seq<float> = seq [||]

  member this.Read(filename:string) =
    let a = File.ReadAllBytes(filename)
    if a.Length < 40 then
      raise(WavUnknownFormat)
    if a.[ 0.. 3] <> Encoding.ASCII.GetBytes("RIFF")
    || a.[ 8..11] <> Encoding.ASCII.GetBytes("WAVE")
    || a.[12..15] <> Encoding.ASCII.GetBytes("fmt ") then
      raise(WavUnknownFormat)

    let ChunkSize    = BitConverter.ToInt32(a.[4..7], 0)
    let SubChunk1Size = BitConverter.ToInt32(a.[16..19], 0)
    let AudioFormatVal = BitConverter.ToInt16(a.[20..21], 0)
    AudioFormat <- match AudioFormatVal with
                   | 1s -> WavPCM
                   | _ -> raise(WavUnknownFormat)

    NumChannels <- int(BitConverter.ToInt16(a.[22..23], 0))
    if NumChannels < 1 || NumChannels > 2 then raise(WavUnknownNumChannels)

    SampleRate <- int(BitConverter.ToInt32(a.[24..27], 0))

    BitsPerSample <- int(BitConverter.ToInt16(a.[34..35], 0))
    if BitsPerSample % 8 <> 0 then raise(WavUnknownBitsPerSample)

    let SubChunk2Start = 12 + 8 + SubChunk1Size
    if a.[SubChunk2Start..(SubChunk2Start+3)] <>
       Encoding.ASCII.GetBytes("data") then
      raise(WavUnknownFormat)

    let SubChunk2Size = BitConverter.ToInt32(
                          a.[(SubChunk2Start+4)..(SubChunk2Start+7)], 0)
    let data = a.[(SubChunk2Start+8)..(SubChunk2Start+8+SubChunk2Size-1)]
    Samples <- match BitsPerSample with
               | 16 -> data |> breakBy 2 |> Seq.map Sample16BitLE2Float
               | _ -> raise(WavUnknownBitsPerSample)

  member this.Write(filename:string) =
    let NumSamples = Seq.length Samples

    let mutable bytes = Encoding.ASCII.GetBytes "RIFF"
    bytes <- Array.append bytes (BitConverter.GetBytes
      (44 + (NumSamples * BitsPerSample/8)))
    bytes <- Array.append bytes (Encoding.ASCII.GetBytes "WAVE")

    bytes <- Array.append bytes (Encoding.ASCII.GetBytes "fmt ")
    match AudioFormat with
    | WavPCM ->
      bytes <- Array.append bytes (BitConverter.GetBytes 16)
      bytes <- Array.append bytes (BitConverter.GetBytes (int16 1))
      bytes <- Array.append bytes (BitConverter.GetBytes (int16 NumChannels))
      bytes <- Array.append bytes (BitConverter.GetBytes SampleRate)
      bytes <- Array.append bytes (BitConverter.GetBytes
        (SampleRate*NumChannels*BitsPerSample/8))
      bytes <- Array.append bytes (BitConverter.GetBytes
        (int16 (NumChannels*BitsPerSample/8)))
      bytes <- Array.append bytes (BitConverter.GetBytes (int16 BitsPerSample))

      bytes <- Array.append bytes (Encoding.ASCII.GetBytes "data")
      bytes <- Array.append bytes (BitConverter.GetBytes
        (NumSamples * NumChannels * BitsPerSample/8))

      let data = Samples |> Seq.map Float2Sample16BitLE
                 |> Seq.collect (fun i -> i) |> Seq.toArray
      bytes <- Array.append bytes data

    File.WriteAllBytes(filename, bytes)

  member this.Print():unit =
    printfn "RIFF ChunkSize= ";

    printf "WAVE AudioFormat=";
    if AudioFormat = WavPCM then printf "PCM "
    else printf "unknown "

    printfn "NumChannels=%d SampleRate=%d BitsPerSample=%d "
      NumChannels SampleRate BitsPerSample

    let NumSamples = Seq.length Samples
    printfn "%d samples, %dmin %ds %dms playtime" NumSamples
      (NumSamples/SampleRate/60)
      ((NumSamples/SampleRate)%60)
      (((NumSamples%SampleRate)*1000)/SampleRate)

  member this.GetSamples() = Samples

  member this.LowPass(samples) =
    // Simple sliding average window filter
    let window_size = 32
    let xcoeff = [| for i in 1..window_size do yield 1.0/float(window_size) |]
    let buffer = ref [|for i in 1..(Array.length xcoeff) do yield 0.0|]
    Seq.map (fun s ->
      buffer := Array.append (!buffer).[1..((Array.length !buffer)-1)] [|s|]
      Array.map2 (fun x b -> x*b) xcoeff !buffer |> Array.sum
      ) samples

  member this.Filter(filter):unit =
    match filter with
    | "none" -> ()
    | "lowpass" ->  Samples <- this.LowPass(Samples)
    | _ -> raise WavUnknownFilter

let args = System.Environment.GetCommandLineArgs()
if (Array.length args) = 4 then
  let filter = args.[1]
  let infilename = args.[2]
  let outfilename = args.[3]
  let w = new WavFile()
  w.Read(infilename)
  w.Print()
  w.Filter(filter)
  w.Write(outfilename)
else
  printf "wav2.exe <filter:none,sinc> <input filename.wav> <output filename.wav>\n"

Invocation:

~> fsc.exe wav2.fs
Microsoft (R) F# 2.0 Compiler build 2.0.0.0
Copyright (c) Microsoft Corporation. All Rights Reserved.
~> time ./wav2.exe lowpass /usr/share/sounds/alsa/test.wav out.wav
RIFF ChunkSize= 
WAVE AudioFormat=PCM NumChannels=2 SampleRate=44100 BitsPerSample=16 
670320 samples, 0min 15s 200ms playtime

real    0m4.253s
user    0m4.221s
sys     0m0.095s

Further reading:

Advertisements
Posted in Algorithms, IO | Leave a comment

Reading and writing WAV audio files

The following example shows simple reading and writing (and later processing) of WAV audio files using File.ReadAllBytes, File.WriteAllBytes and BitConverter.

> open System.IO;;
> File.ReadAllBytes;;
val it : string -> byte [] = <fun:clo@4>
> File.WriteAllBytes;;
val it : string * byte [] -> unit = <fun:clo@5-1>
> BitConverter.ToInt16;;
val it : byte [] * int -> int16 = <fun:clo@6-2>
> let a = File.ReadAllBytes("testin.wav");;
val it : byte [] =
  [|82uy; 73uy; 70uy; 70uy; 4uy; 117uy; 20uy; 0uy; 87uy; 65uy; 86uy; 69uy;
    102uy; 109uy; 116uy; 32uy; 16uy; 0uy; 0uy; 0uy; 1uy; 0uy; 2uy; 0uy; 68uy;
    172uy; 0uy; 0uy; 16uy; 177uy; 2uy; 0uy; 4uy; 0uy; 16uy; 0uy; 100uy; 97uy;
    116uy; 97uy; 224uy; 116uy; 20uy; 0uy; 154uy; 255uy; 170uy; 255uy; 116uy;
    255uy; 136uy; 255uy; 97uy; 255uy; 117uy; 255uy; 90uy; 255uy; 107uy; 255uy;
    76uy; 255uy; 101uy; 255uy; 76uy; 255uy; 104uy; 255uy; 78uy; 255uy; 103uy;
    255uy; 108uy; 255uy; 101uy; 255uy; 128uy; 255uy; 104uy; 255uy; 111uy;
    255uy; 93uy; 255uy; 87uy; 255uy; 75uy; 255uy; 66uy; 255uy; 50uy; 255uy;
    63uy; 255uy; 23uy; 255uy; 22uy; 255uy; 249uy; 254uy; ...|]

File.ReadAllBytes creates an array which contains all bytes of the file. The binary content must then be identified and transferred to an internal data structure.

> open System.Text;;
> a.[ 0.. 3] = Encoding.ASCII.GetBytes("RIFF");;
val it : bool = true

Individual parameters are parsed by passing sliced parts of the array to BitConverter.ToInt32 and BitConverter.ToInt16. Endianness is applied according host settings.

> BitConverter.IsLittleEndian;;  
val it : bool = true
> let ChunkSize    = BitConverter.ToInt32(a.[4..7], 0);;

val ChunkSize : int = 1340676

> let SubChunk1Size = BitConverter.ToInt32(a.[16..19], 0);;

val SubChunk1Size : int = 16

> let AudioFormatVal = BitConverter.ToInt16(a.[20..21], 0);;

val AudioFormatVal : int16 = 1s

To generate an array of samples it is necessary to combine two bytes into one 16 bit sample integer. The sequence function Seq.breakBy is used to split the array into an array of 2 element arrays (see also here), these two element arrays are then converted into 16 bit integers.

> #r "seq1.dll";;

--> Referenced '/home/ootoo/Projekte/FSharp/2sharp4u/2011-10-29 Reading and writing WAV audio files/seq1.dll'

> open Microsoft.FSharp.Collections;;
> let data = a.[(SubChunk2Start+8)..(SubChunk2Start+8+SubChunk2Size-1)];;

val data : byte [] =
  [|154uy; 255uy; 170uy; 255uy; 116uy; 255uy; 136uy; 255uy; 97uy; 255uy; 117uy;
    255uy; 90uy; 255uy; 107uy; 255uy; 76uy; 255uy; 101uy; 255uy; 76uy; 255uy;
    104uy; 255uy; 78uy; 255uy; 103uy; 255uy; 108uy; 255uy; 101uy; 255uy; 128uy;
    255uy; 104uy; 255uy; 111uy; 255uy; 93uy; 255uy; 87uy; 255uy; 75uy; 255uy;
    66uy; 255uy; 50uy; 255uy; 63uy; 255uy; 23uy; 255uy; 22uy; 255uy; 249uy;
    254uy; 239uy; 254uy; 220uy; 254uy; 226uy; 254uy; 209uy; 254uy; 242uy;
    254uy; 215uy; 254uy; 236uy; 254uy; 209uy; 254uy; 225uy; 254uy; 213uy;
    254uy; 218uy; 254uy; 219uy; 254uy; 225uy; 254uy; 235uy; 254uy; 237uy;
    254uy; 0uy; 255uy; 251uy; 254uy; 18uy; 255uy; 6uy; 255uy; 28uy; 255uy; 7uy;
    255uy; 22uy; 255uy; ...|]
> let Sample16BitLE2Int i = int(BitConverter.ToInt16((Seq.toArray i), 0));;

val Sample16BitLE2Int : seq<byte> -> int

> let Samples = data |> Seq.breakByV1 2 |> Seq.map Sample16BitLE2Int;;

val Samples : seq<int>

Below is the complete source code:

open System
open System.IO
open System.Text

// https://ccrma.stanford.edu/courses/422/projects/WaveFormat/

type WavAudioFormat = WavPCM
exception WavUnknownFormat
exception WavUnknownNumChannels
exception WavUnknownBitsPerSample

let Sample16BitLE2Int i = int(BitConverter.ToInt16((Seq.toArray i), 0))
let Int2Sample16BitLE s = BitConverter.GetBytes(int16(s))

let breakBy windowSize (source: seq<_>) =    
    seq { let arr = Array.create windowSize (Seq.nth 0 source)
          let i = ref 0
          use e = source.GetEnumerator()
          while e.MoveNext() do
              arr.[!i] <- e.Current
              i := !i + 1
              if !i = windowSize then
                yield Array.init windowSize (fun j -> arr.[j])
                i := 0
        }

type WavFile() =
  let mutable AudioFormat:WavAudioFormat = WavPCM
  let mutable NumChannels:int = 0
  let mutable SampleRate:int = 0
  let mutable BitsPerSample:int = 0
  let mutable Samples:seq<int> = seq [||]

  member this.Read(filename:string) =
    let a = File.ReadAllBytes(filename)
    if a.Length < 40 then
      raise(WavUnknownFormat)
    if a.[ 0.. 3] <> Encoding.ASCII.GetBytes("RIFF")
    || a.[ 8..11] <> Encoding.ASCII.GetBytes("WAVE")
    || a.[12..15] <> Encoding.ASCII.GetBytes("fmt ") then
      raise(WavUnknownFormat)

    let ChunkSize    = BitConverter.ToInt32(a.[4..7], 0)
    let SubChunk1Size = BitConverter.ToInt32(a.[16..19], 0)
    let AudioFormatVal = BitConverter.ToInt16(a.[20..21], 0)
    AudioFormat <- match AudioFormatVal with
                   | 1s -> WavPCM
                   | _ -> raise(WavUnknownFormat)

    NumChannels <- int(BitConverter.ToInt16(a.[22..23], 0))
    if NumChannels < 1 || NumChannels > 2 then raise(WavUnknownNumChannels)

    SampleRate <- int(BitConverter.ToInt32(a.[24..27], 0))

    BitsPerSample <- int(BitConverter.ToInt16(a.[34..35], 0))
    if BitsPerSample % 8 <> 0 then raise(WavUnknownBitsPerSample)

    let SubChunk2Start = 12 + 8 + SubChunk1Size
    if a.[SubChunk2Start..(SubChunk2Start+3)] <>
       Encoding.ASCII.GetBytes("data") then
      raise(WavUnknownFormat)

    let SubChunk2Size = BitConverter.ToInt32(
                          a.[(SubChunk2Start+4)..(SubChunk2Start+7)], 0)
    let data = a.[(SubChunk2Start+8)..(SubChunk2Start+8+SubChunk2Size-1)]
    Samples <- match BitsPerSample with
               | 16 -> data |> breakBy 2 |> Seq.map Sample16BitLE2Int
               | _ -> raise(WavUnknownBitsPerSample)

  member this.Write(filename:string) =
    let NumSamples = Seq.length Samples

    let mutable bytes = Encoding.ASCII.GetBytes "RIFF"
    bytes <- Array.append bytes (BitConverter.GetBytes
      (44 + (NumSamples * BitsPerSample/8)))
    bytes <- Array.append bytes (Encoding.ASCII.GetBytes "WAVE")

    bytes <- Array.append bytes (Encoding.ASCII.GetBytes "fmt ")
    match AudioFormat with
    | WavPCM ->
      bytes <- Array.append bytes (BitConverter.GetBytes 16)
      bytes <- Array.append bytes (BitConverter.GetBytes (int16 1))
      bytes <- Array.append bytes (BitConverter.GetBytes (int16 NumChannels))
      bytes <- Array.append bytes (BitConverter.GetBytes SampleRate)
      bytes <- Array.append bytes (BitConverter.GetBytes
        (SampleRate*NumChannels*BitsPerSample/8))
      bytes <- Array.append bytes (BitConverter.GetBytes
        (int16 (NumChannels*BitsPerSample/8)))
      bytes <- Array.append bytes (BitConverter.GetBytes (int16 BitsPerSample))

      bytes <- Array.append bytes (Encoding.ASCII.GetBytes "data")
      bytes <- Array.append bytes (BitConverter.GetBytes
        (NumSamples * NumChannels * BitsPerSample/8))

      let data = Samples |> Seq.map Int2Sample16BitLE
                 |> Seq.collect (fun i -> i) |> Seq.toArray
      bytes <- Array.append bytes data

    File.WriteAllBytes(filename, bytes)

  member this.Print():unit =
    printfn "RIFF ChunkSize= ";

    printf "WAVE AudioFormat=";
    if AudioFormat = WavPCM then printf "PCM "
    else printf "unknown "

    printfn "NumChannels=%d SampleRate=%d BitsPerSample=%d "
      NumChannels SampleRate BitsPerSample

    let NumSamples = Seq.length Samples
    printfn "%d samples, %dmin %ds %dms playtime" NumSamples
      (NumSamples/SampleRate/60)
      ((NumSamples/SampleRate)%60)
      (((NumSamples%SampleRate)*1000)/SampleRate)

  member this.GetSamples() = Samples

let args = System.Environment.GetCommandLineArgs()
if (Array.length args) = 3 then
  let infilename = args.[1]
  let outfilename = args.[2]
  let w = new WavFile()
  w.Read(infilename)
  w.Print()
  w.Write(outfilename)
else
  printf "wav1.exe <input filename.wav> <output filename.wav>\n"
Posted in Getting Started, IO | 1 Comment

Efficient Sequence Operators: Seq.breakBy

I wanted to create a sequence function which would break a sequence into a number of equally sized chunks (e.g. to create a sequence of 4 byte arrays out of a byte stream to pass it to System.BitConverter). Here is my first attempt

~> cat seq1.fs
namespace Microsoft.FSharp.Collections
  module Seq =
    let breakByV1 windowSize (source:seq<_>) =
      source |> Seq.windowed windowSize
      |> Seq.mapi (fun i j -> if i%windowSize=0 then Some j else None)
      |> Seq.choose (fun i -> i)
~> fsc.exe --target:library seq1.fs
~> fsi.exe

Microsoft (R) F# 2.0 Interactive build 2.0.0.0
Copyright (c) Microsoft Corporation. All Rights Reserved.

For help type #help;;
> #r "seq1.dll";;

--> Referenced 'seq1.dll'

> open Microsoft.FSharp.Collections;;
> {1..100} |> Seq.breakByV1 3;;
val it : seq<int []> =
  seq [[|1; 2; 3|]; [|4; 5; 6|]; [|7; 8; 9|]; [|10; 11; 12|]; ...]

The file ‘seq1.fs’ module defines a function ‘Seq.breakByV1’ inside namespace ‘Microsoft.FSharp.Collections’ and module ‘Seq’. This allows naming and access of the as ‘Seq.breakByV1’ function as it were part of the original Microsoft Seq module.

Let’s do some timing on the first implementation:

> #time "on";;

--> Timing now on

> let n=1000000;; let chunk=100;;
> {1..n} |> Seq.breakByV1 chunk |> Seq.nth (n/chunk-1);;
Real: 00:00:02.588, CPU: 00:00:00.000, GC gen0: 39
val it : int [] = 
[|999901; 999902; 999903; 999904; 999905; 999906; 999907; 999908; 999909;
  999910; 999911; 999912; 999913; 999914; 999915; 999916; 999917; 999918;
  999919; 999920; 999921; 999922; 999923; 999924; 999925; 999926; 999927;
  999928; 999929; 999930; 999931; 999932; 999933; 999934; 999935; 999936;
  999937; 999938; 999939; 999940; 999941; 999942; 999943; 999944; 999945;
  999946; 999947; 999948; 999949; 999950; 999951; 999952; 999953; 999954;
  999955; 999956; 999957; 999958; 999959; 999960; 999961; 999962; 999963;
  999964; 999965; 999966; 999967; 999968; 999969; 999970; 999971; 999972;
  999973; 999974; 999975; 999976; 999977; 999978; 999979; 999980; 999981;
  999982; 999983; 999984; 999985; 999986; 999987; 999988; 999989; 999990;
  999991; 999992; 999993; 999994; 999995; 999996; 999997; 999998; 999999;
  1000000|]

As you can see this implementation takes approximately 2.5 seconds to cut a sequence with 1 million entries into chunks with 100 elements each. However, this implementation is using 3 sequence operators (Seq.windowed, Seq.mapi, Seq.choose) in a not too straightforward fashion, therefore, some optimization should be possible.

Since the F# compiler and library sources were released into open source, there is a good opportunity to learn from the core library implementations. Therefore, let’s have look at the implementation of ‘Seq.windowed’ inside seq.fs of the F# Compiler (get it here):

~> cat compiler/2.0/Nov2010/src/fsharp/FSharp.Core/seq.fs
...
       // windowed : int -> seq<'T> -> seq<'T[]>
        [<CompiledName("Windowed")>]
        let windowed windowSize (source: seq<_>) =    
            checkNonNull "source" source
            if windowSize <= 0 then invalidArg "windowSize" (SR.GetString(SR.inputMustBeNonNegative))
            seq { let arr = Microsoft.FSharp.Primitives.Basics.Array.zeroCreateUnchecked windowSize
                  let r = ref (windowSize-1)
                  let i = ref 0
                  use e = source.GetEnumerator()
                  while e.MoveNext() do
                      arr.[!i] <- e.Current
                      i := (!i + 1) % windowSize
                      if !r = 0 then
                          yield Array.init windowSize (fun j -> arr.[(!i+j) % windowSize])
                      else
                      r := (!r - 1) }
...

The Microsoft implementation is implemented in a not too functional fashion, with reference values and assignments, however, speed improvement justifies this low level plumbing as we will see in a moment.

The Seq.breakByV2 function is an implementation following the structure of Seq.Windowed as seen above.

~> cat seq1.fs
namespace Microsoft.FSharp.Collections
  module Seq =
    let breakByV1 windowSize (source:seq<_>) =
      source |> Seq.windowed windowSize
      |> Seq.mapi (fun i j -> if i%windowSize=0 then Some j else None)
      |> Seq.choose (fun i -> i)

    let breakByV2 windowSize (source: seq<_>) =
      seq { let arr = Array.create windowSize (Seq.nth 0 source)
            let i = ref 0
            use e = source.GetEnumerator()
            while e.MoveNext() do
              arr.[!i] <- e.Current
              i := !i + 1
              if !i = windowSize then
                yield Array.init windowSize (fun j -> arr.[j])
                i := 0
          }
~> fsc.exe --target:library seq1.fs
~> fsi.exe

Microsoft (R) F# 2.0 Interactive build 2.0.0.0
Copyright (c) Microsoft Corporation. All Rights Reserved.

For help type #help;;
> #r "seq1.dll";;

--> Referenced 'seq1.dll'

> open Microsoft.FSharp.Collections;;

Let’s do the same benchmark as done before on ‘Seq.breakByV1’:

> {1..n} |> Seq.breakByV2 chunk |> Seq.nth (n/chunk-1);;
Real: 00:00:00.118, CPU: 00:00:00.000, GC gen0: 1
val it : int [] =
  [|999901; 999902; 999903; 999904; 999905; 999906; 999907; 999908; 999909;
    999910; 999911; 999912; 999913; 999914; 999915; 999916; 999917; 999918;
    999919; 999920; 999921; 999922; 999923; 999924; 999925; 999926; 999927;
    999928; 999929; 999930; 999931; 999932; 999933; 999934; 999935; 999936;
    999937; 999938; 999939; 999940; 999941; 999942; 999943; 999944; 999945;
    999946; 999947; 999948; 999949; 999950; 999951; 999952; 999953; 999954;
    999955; 999956; 999957; 999958; 999959; 999960; 999961; 999962; 999963;
    999964; 999965; 999966; 999967; 999968; 999969; 999970; 999971; 999972;
    999973; 999974; 999975; 999976; 999977; 999978; 999979; 999980; 999981;
    999982; 999983; 999984; 999985; 999986; 999987; 999988; 999989; 999990;
    999991; 999992; 999993; 999994; 999995; 999996; 999997; 999998; 999999;
    1000000|]

The new implementation Seq.breakByV2 just takes 118 miliseconds, thats an improvement of more than 20 times compared to Seq.breakByV1.

Further reading:

Posted in Algorithms, Language | 1 Comment

Register Mono / F# executables on Linux

It is kind of inconvenient to have to type ‘mono foo.exe’ every time you want to execute a mono executable on Linux. However, Linux offers a kernel module ‘binfmt_misc’ which allows to register customer binary formats and define how they shall be executed. Below is a daemon script for OpenSuSE 11.4 which registers automatically during startup Mono exeutables (derived from /etc/init.d/java_binfmt_misc).

You need to perform the following steps to register Mono (includes F#) executables:

1. Create /etc/init.d/mono_binfmt_misc from text below

2. Create link to script

> ln -s /etc/init.d/mono_binfmt_misc /usr/sbin/rcmono_binfmt_misc

3. Start daemon

> rcmono_binfmt_misc start

4. Activate daemon for default runlevels (3 & 5)

> chkconfig mono_binfmt_misc

5. Test if it works by calling a mono executable (without calling mono explicitly):

> /usr/lib/mono/4.0/xsp4.exe
xsp4
Listening on address: 0.0.0.0
Root directory: ./2sharp4u
Listening on port: 8080 (non-secure)
Hit Return to stop the server.

File contents /etc/init.d/mono_binfmt_misc:

#!/bin/sh
#
#     Template SUSE system startup script for example service/daemon mono.binfmt_misc
#     Copyright (C) 1995--2005  Kurt Garloff, SUSE / Novell Inc.
#          
#     This library is free software; you can redistribute it and/or modify it
#     under the terms of the GNU Lesser General Public License as published by
#     the Free Software Foundation; either version 2.1 of the License, or (at
#     your option) any later version.
#                             
#     This library is distributed in the hope that it will be useful, but
#     WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#     Lesser General Public License for more details.
#      
#     You should have received a copy of the GNU Lesser General Public
#     License along with this library; if not, write to the Free Software
#     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307,
#     USA.
#
# /etc/init.d/mono_binfmt_misc
#   and its symbolic link
# /(usr/)sbin/rcmono_binfmt_misc
#
# Template system startup script for some example service/daemon mono_binfmt_misc
#
# LSB compatible service control script; see http://www.linuxbase.org/spec/
#
#
### BEGIN INIT INFO
# Provides:          mono_binfmt_misc
# Required-Start:    $remote_fs
# Should-Start:
# Required-Stop:     $remote_fs
# Should-Stop:
# Default-Start:     3 5
# Default-Stop:      0 1 2 6
# Short-Description: enables the system to automatically execute mono *.exe files
# Description:       Start mono_infmt_misc to allow the execution of mono programs with *.exe files
### END INIT INFO

# see #271198
mono_binfmt_misc_status ()
{
    [ ! -e /proc/sys/fs/binfmt_misc/status ] && return 4
    [ `cat /proc/sys/fs/binfmt_misc/status` != "enabled" ] && return 3

    for state in CLR
    do
        if [ ! -e /proc/sys/fs/binfmt_misc/$state ]; then
            echo Warning /proc/sys/fs/binfmt_misc/$state is not present
            return 3
        fi

        if [ `head -n 1 /proc/sys/fs/binfmt_misc/$state` != "enabled" ]
        then
            echo Warning /proc/sys/fs/binfmt_misc/$state is not enabled
        fi
    done

    return 0
}

# Check for missing binaries (stale symlinks should not happen)
# Note: Special treatment of stop for LSB conformance
for file in /usr/bin/mono
do
    test -x $file || { echo "$file not installed";
    if [ "$1" = "stop" ]; then exit 0;
    else exit 5; fi; }
done

# Check for existence of needed config file and read it
#mono.binfmt_misc_CONFIG=/etc/sysconfig/mono.binfmt_misc
#test -r $mono.binfmt_misc_CONFIG || { echo "$mono.binfmt_misc_CONFIG not existing";
#       if [ "$1" = "stop" ]; then exit 0;
#       else exit 6; fi; }

# Read config
#. $mono.binfmt_misc_CONFIG

# Source LSB init functions
# providing start_daemon, killproc, pidofproc,
# log_success_msg, log_failure_msg and log_warning_msg.
# This is currently not used by UnitedLinux based distributions and
# not needed for init scripts for UnitedLinux only. If it is used,
# the functions from rc.status should not be sourced or used.
#. /lib/lsb/init-functions

# Shell functions sourced from /etc/rc.status:
#      rc_check         check and set local and overall rc status
#      rc_status        check and set local and overall rc status
#      rc_status -v     be verbose in local rc status and clear it afterwards
#      rc_status -v -r  ditto and clear both the local and overall rc status
#      rc_status -s     display "skipped" and exit with status 3
#      rc_status -u     display "unused" and exit with status 3
#      rc_failed        set local and overall rc status to failed
#      rc_failed <num>  set local and overall rc status to <num>
#      rc_reset         clear both the local and overall rc status
#      rc_exit          exit appropriate to overall rc status
#      rc_active        checks whether a service is activated by symlinks
. /etc/rc.status

# Reset status of this service
rc_reset

# Return values acc. to LSB for all commands but status:
# 0       - success
# 1       - generic or unspecified error
# 2       - invalid or excess argument(s)
# 3       - unimplemented feature (e.g. "reload")
# 4       - user had insufficient privileges
# 5       - program is not installed
# 6       - program is not configured
# 7       - program is not running
# 8--199  - reserved (8--99 LSB, 100--149 distrib, 150--199 appl)
#
# Note that starting an already running service, stopping
# or restarting a not-running service as well as the restart
# with force-reload (in case signaling is not supported) are
# considered a success.

case "$1" in
    start)
    echo -n "Starting mono.binfmt_misc "

    # Insert BINFMT_MISC module into the kernel
    if [ ! -e /proc/sys/fs/binfmt_misc/register ]; then
        if [ -x /sbin/modprobe ]; then
            /sbin/modprobe binfmt_misc
            # Some distributions, like Fedora Core, perform
            # the following command automatically when the
            # binfmt_misc module is loaded into the kernel.
            # Thus, it is possible that the following line
            # is not needed at all. Look at /etc/modprobe.conf
            # to check whether this is applicable or not.
            if ! mount | grep -q binfmt
            then
                mount -t binfmt_misc none /proc/sys/fs/binfmt_misc
            fi
        fi
    fi

    # Register support for *.exe files
    if [ -e /proc/sys/fs/binfmt_misc/register ]; then
        # support for mono applications:
        if [ -e /proc/sys/fs/binfmt_misc/mono ]
        then
            echo 1 > /proc/sys/fs/binfmt_misc/mono
        else
            echo ':CLR:M::MZ::/usr/bin/mono:' > /proc/sys/fs/binfmt_misc/register
        fi
        echo 1 > /proc/sys/fs/binfmt_misc/status
        ## support for mono executable:
    fi

    # Remember status and be verbose
    rc_status -v
    ;;
    stop)
    echo -n "Shutting down mono_binfmt_misc "
    # De-register support for *.exe files
    if [ -e /proc/sys/fs/binfmt_misc/register ]; then
        # support for mono applications:
        if [ -e /proc/sys/fs/binfmt_misc/CLR ]; then
            echo -1 > /proc/sys/fs/binfmt_misc/CLR
        fi
    fi

    # Remember status and be verbose
    rc_status -v
    ;;
    try-restart|condrestart)
    ## Do a restart only if the service was active before.
    ## Note: try-restart is now part of LSB (as of 1.9).
    ## RH has a similar command named condrestart.
    if test "$1" = "condrestart"; then
        echo "${attn} Use try-restart ${done}(LSB)${attn} rather than condrestart ${warn}(RH)${norm}"
    fi
    $0 status
    if test $? = 0; then
        $0 restart
    else
        rc_reset        # Not running is not a failure.
    fi
    # Remember status and be quiet
    rc_status
    ;;
    restart)
    ## Stop the service and regardless of whether it was
    ## running or not, start it again.
    $0 stop
    $0 start

    # Remember status and be quiet
    rc_status
    ;;

    reload)
    rc_failed 3
    ;;

    ## removed see #271198

    #    force-reload)
    #   ## Signal the daemon to reload its config. Most daemons
    #   ## do this on signal 1 (SIGHUP).
    #   ## If it does not support it, restart the service if it
    #   ## is running.
    #
    #   echo -n "Reload service mono.binfmt_misc "
    #   ## if it supports it:
    #   echo /sbin/killproc -HUP $mono.binfmt_misc_BIN
    #   /sbin/killproc -HUP $mono.binfmt_misc_BIN
    #   #touch /var/run/mono.binfmt_misc.pid
    #   rc_status -v
    #
    #   ## Otherwise:
    #   #$0 try-restart
    #   #rc_status
    #   ;;
    #    reload)
    #   ## Like force-reload, but if daemon does not support
    #   ## signaling, do nothing (!)
    #
    #   # If it supports signaling:
    #   echo -n "Reload service mono.binfmt_misc "
    #   /sbin/killproc -HUP $mono.binfmt_misc_BIN
    #   #touch /var/run/mono.binfmt_misc.pid
    #   rc_status -v
    #
    #   ## Otherwise if it does not support reload:
    #   #rc_failed 3
    #   #rc_status -v
    #   ;;
    status)
    echo -n "Checking for service mono_binfmt_misc "
    ## Check status with checkproc(8), if process is running
    ## checkproc will return with exit status 0.

    # Return value is slightly different for the status command:
    # 0 - service up and running
    # 1 - service dead, but /var/run/  pid  file exists
    # 2 - service dead, but /var/lock/ lock file exists
    # 3 - service not running (unused)
    # 4 - service status unknown :-(
    # 5--199 reserved (5--99 LSB, 100--149 distro, 150--199 appl.)

    # this function returns the LSB compliants status code
    # see #271198
    mono_binfmt_misc_status
    # NOTE: rc_status knows that we called this init script with
    # "status" option and adapts its messages accordingly.
    rc_status -v
    ;;
    #    probe)
    #   ## Optional: Probe for the necessity of a reload, print out the
    #   ## argument to this init script which is required for a reload.
    #   ## Note: probe is not (yet) part of LSB (as of 1.9)
    #
    #   test /etc/mono.binfmt_misc/mono.binfmt_misc.conf -nt /var/run/mono.binfmt_misc.pid && echo reload
    #   ;;
    *)
    echo "Usage: $0 {start|stop|status|try-restart|restart}"
    exit 1
    ;;
esac
rc_exit
Posted in Installation | 3 Comments

Mono 2.10 released – includes F#

Mono 2.10 has been released. Mono now includes F#, look here for details.

Major highlights:

 

Posted in General | Leave a comment

Basic MySQL Transactions

The following example shows how to access a MySQL database with F# on Mono. To try it out, please install the MySQL NET/Connector in advance.

Firstly, lets create an example database and a mysql user account.

~> mysql -u root -p
Enter password:
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 35
Server version: 5.1.46-log SUSE MySQL RPM

Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
This software comes with ABSOLUTELY NO WARRANTY. This is free software,
and you are welcome to modify and redistribute it under the GPL v2 license

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> create database test;
Query OK, 1 row affected (0.00 sec)
mysql> create user "2sharp4u"@"localhost" identified by "fsharp";
Query OK, 0 rows affected (0.00 sec)
mysql> grant all on test to 2sharp4u;
Query OK, 0 rows affected (0.00 sec)
mysql> quit
Bye

Secondly, we may apply our F# code to open, edit and retrieve the data out of this database.

open MySql.Data.MySqlClient

let conn = new MySqlConnection("Server=localhost;"+
 "Uid=2sharp4u;"+
 "Pwd=fsharp;"+
 "Database=test")
conn.Open()

let cmd1 = new MySqlCommand("CREATE TABLE presidents (
 ID INT NOT NULL,
 FirstName VARCHAR(30) NOT NULL,
 LastName VARCHAR(30) NOT NULL,
 TookOffice datetime,
 LeftOffice datetime,
 PRIMARY KEY (ID))", conn)
cmd1.ExecuteNonQuery() |> ignore

let cmd2 = new MySqlCommand("INSERT INTO presidents
 (ID, FirstName, LastName, TookOffice, LeftOffice)
 VALUES (1, 'George', 'Washington', '30.04.1789', '04.03.1797')", conn)
cmd2.ExecuteNonQuery() |> ignore

let cmd3 = new MySqlCommand("INSERT INTO presidents
 (ID, FirstName, LastName, TookOffice, LeftOffice)
 VALUES (2, 'John', 'Adams', '04.03.1797', '04.03.1801')", conn)
cmd3.ExecuteNonQuery() |> ignore

let cmd4 = new MySqlCommand("INSERT INTO presidents
 (ID, FirstName, LastName, TookOffice, LeftOffice)
 VALUES (3, 'Thomas', 'Jefferson', '04.03.1801', '04.03.1809')", conn)
cmd4.ExecuteNonQuery() |> ignore

let qcmd = new MySqlCommand("SELECT FirstName,
LastName FROM presidents", conn)
let reader = qcmd.ExecuteReader()

let s = seq {
 while reader.Read() do
 yield (reader.GetString 0, reader.GetString 1)
 }

s |> Seq.iter (fun (firstname, lastname) ->
 printfn "%s %s" firstname lastname)

conn.Close()

Compile and execute the source file.

~> fsc -r MySql.Data.dll mysql1.fs
Microsoft (R) F# 2.0 Compiler build 2.0.0.0
Copyright (c) Microsoft Corporation. All Rights Reserved.
~> mono mysql1.exe
George Washington
John Adams
Thomas Jefferson
~>
Posted in Database, Getting Started | Leave a comment

Installing MySql Connector for Mono

DOTNET and Mono by default only support database connections to the Microsoft SQL Server through System.Data.SqlClient module. To make a connection to the MySQL server you need a connector module from MySQL.

Proceed with the following steps to obtain and install the MySQL .NET Connector:

1. Downloading MySQL .NET Connector

Either from MySQL (requires registration):

http://www.mysql.com/downloads/mirror.php?id=399565

or directly from one of the mirros:

ftp://ftp.gwdg.de/pub/misc/mysql/Downloads/Connector-Net/mysql-connector-net-6.3.6-noinstall.zip

3. Unzip the content into a directory

~> md connector
~> cd connector
~/connector> unzip ../mysql-connector-net-6.3.6-noinstall.zip
~/connector> ls
CHANGES  COPYING  MySql.Data.chm  README  Release Notes.txt  v2  v4

4. Change names of assemblies to proper names

~/connector> cd v2
~/connector/v2/ mv mysql.data.cf.dll MySql.Data.Cf.dll
~/connector/v2> mv mysql.data.dll MySql.Data.dll
~/connector/v2> mv mysql.data.entity.dll MySql.Data.Entity.dll
~/connector/v2> mv mysql.visualstudio.dll MySql.VisualStudio.dll
~/connector/v2> mv mysql.web.dll MySql.Web.dll
~/connector/v2> cd ../v4
~/connector/v4> mv mysql.data.dll MySql.Data.dll
~/connector/v4> mv mysql.data.entity.dll MySql.Data.Entity.dll
~/connector/v4> mv mysql.visualstudio.dll MySql.VisualStudio.dll
~/connector/v4> mv mysql.web.dll MySql.Web.dll

5. Copy assemblies into library directory of your mono installation

~/connector/v4> su
/home/user/connector/v4 # cp MySql.* /usr/lib/mono/4.0/.
/home/user/connector/v4 # cd ../v2
/home/user/connector/v2 # cp MySql.* /usr/lib/mono/2.0/.

6. Register the installed assemblies in the global assembly cache

~ # cd /usr/lib/mono/4.0
/usr/lib/mono/4.0 # gacutil -i MySql.Data.dll
Installed MySql.Data.dll into the gac (/usr/lib/mono/gac)
/usr/lib/mono/4.0 # gacutil -i MySql.Data.Entity.dll
Installed MySql.Data.Entity.dll into the gac (/usr/lib/mono/gac)
/usr/lib/mono/4.0 # gacutil -i MySql.VisualStudio.dll
Installed MySql.VisualStudio.dll into the gac (/usr/lib/mono/gac)
/usr/lib/mono/4.0 # gacutil -i MySql.Web.dll
Installed MySql.Web.dll into the gac (/usr/lib/mono/gac)
/usr/lib/mono/4.0 # cd ../2.0
/usr/lib/mono/2.0 # gacutil -i MySql.Data.Cf.dll
Installed MySql.Data.Cf.dll into the gac (/usr/lib/mono/gac)
/usr/lib/mono/2.0 # gacutil -i MySql.Data.dll
Installed MySql.Data.dll into the gac (/usr/lib/mono/gac)
/usr/lib/mono/2.0 # gacutil -i MySql.Data.Entity.dll
Installed MySql.Data.Entity.dll into the gac (/usr/lib/mono/gac)
/usr/lib/mono/2.0 # gacutil -i MySql.VisualStudio.dll
Installed MySql.VisualStudio.dll into the gac (/usr/lib/mono/gac)
/usr/lib/mono/2.0 # gacutil -i MySql.Web.dll
Installed MySql.Web.dll into the gac (/usr/lib/mono/gac)

Note: For steps 4 (renaming) and 5 (copying), I don’t really understand why these steps are necessary, but on my OpenSuSE 11.3 machine it did only work out that way. Any suggestions to improve this procedure are welcome.

Further reading:

Posted in Database, Installation | 5 Comments

F# is Open Source

Don Syme announced on 4th of November 2010 that the F# compiler and core libraries are released as open source under the Apache 2.0 license, you may download it here.

I think it is unprecedented that a commercially developed languages is released as open source in such an early stage, usually it rather happens when a technology fails to provide sufficient revenues.

Further reading

Posted in General | 1 Comment

Offline

Due to job priorities I will be offline for some time. Thanks for your patience 😉

Posted in General | Leave a comment

ASP.NET Webforms and Event Handling

To create more active server pages as done in the first example, it is necessary to add code and reference elements of a page. The most simple way is to add a script section in the ‘index.aspx’ source. It may be surprising to see that the event handler is defined as a member function of a System.Web.UI.Page class definition.

In the next example we add an event handler for a button click. As usual you need a ‘web.config’ file and a subdirectory bin with with FSharp.Compiler.CodeDom.dll, see in the first example how to do it, the source file in our example should be named index.aspx:

<%@ Page Language="F#" Trace="false" %>

<script runat="server">
member page.mybutton_Click((sender:obj), (e:EventArgs)) =
 page.mybutton.Text <- "Clicked"
</script>

<html>
<body>
<h2>ASP.NET Simple Web Forms Demo</h2>

<form method="post" runat="server">
 button: <asp:button id="mybutton" runat="server" Text="Click Me"
          OnClick="mybutton_Click"/>
</form>

</body>
</html>

When starting the web server xsp2 in the directory of your source file you should see the following content in your web browser when pointing it to ‘http://localhost:8080&#8217;:

When you press the button the text in the button should change to:

One useful feature which you may need in the future is tracing. Change the attribute ‘Trace’ in the first line of the source file from “false” to “true”:

<%@ Page Language="F#" Trace="true" %>

When loading the page it will include additional tracing and timing information:

Further reading:

Posted in Getting Started, Web | 1 Comment