Improving system modules support in swiftpm

Swift package manager supports C system modules by creating a package with module.modulemap file but there are many issues as descibed by Max Howell in his draft proposal (mailing list) like:

varying install locations of system modules
/usr/lib:/usr/local/lib is not always a sufficient -L search path
/usr/include:/usr/local/include is not always a sufficient -I C compiler search path
Installing the system library is left up to the end-user to figure out

The solution involves use of a tool called pkg-config which can be used to ask a system module about its -I search paths and -L flags and most system modules already conform to pkg-config however it does make swiftpm dependent on it for system modules.

Lets try to do this and make a simple Gtk window.

Create a package called CGtk with two files Package.swift and module.modulemap

import PackageDescription

let package = Package(
  name: "CGtk3",
  pkgConfig: "gtk+-3.0",
  providers: [.Brew("gtk+3"), .Apt("gtk+3")]
)

module CGtk [system] {
  header "/usr/local/include/gtk-3.0/gtk/gtk.h"
  link "gtk-3.0"
  export *
}

This is a normal modulemap wrapper package for a system module with two extra things in Package().

pkgConfig - is the name of system module which has to be used while querying pkg-config, the proposal suggested to parse modulemap and infer name from link but looks like they won’t always match.
providers - is an array of enum which will suggest system packagers to use depending upon the system in case the system module is not installed.

These variables have to be added in Package class of PackageDescription module:

public let pkgConfig: String?
public let providers: [SystemPackageProvider]?
...
public enum SystemPackageProvider {
  case Brew(String)
  case Apt(String)
}

Since manifest files is loaded, dumped to TOML and then reconverted back to swift, these should be made TOML convertible.

extension SystemPackageProvider: TOMLConvertible {
  var nameValue: (String, String) {
    switch self {
    case .Brew(let name):
      return ("Brew", name)
    case .Apt(let name):
      return ("Apt", name)
    }
  }
  public func toTOML() -> String {
    let (name, value) = nameValue
    var str = ""
    str += "name = \"\(name)\"\n"
    str += "value = \"\(value)\"\n"
    return str
  }
}

extension Package: TOMLConvertible {
  public func toTOML() -> String {
    ...
    if let pkgConfig = self.pkgConfig {
      result += "pkgConfig = \"\(pkgConfig)\"\n"
    }
    ...
    if let providers = self.providers {
      for provider in providers {
        result += "[[package.providers]]\n"
        result += provider.toTOML()
      }
    }
    ...
  }
}

Also, need to update fromTOML().swift in ManifestParser module to parse the TOML representation.

extension PackageDescription.SystemPackageProvider {
  private static func fromTOML(item: TOMLItem) -> PackageDescription.SystemPackageProvider {
    guard case .Table(let table) = item else { fatalError("unexpected item") }
    guard case .String(let name)? = table.items["name"] else { fatalError("missing name") }
    guard case .String(let value)? = table.items["value"] else { fatalError("missing value") }
    switch name {
    case "Brew":
      return .Brew(value)
    case "Apt":
      return .Apt(value)
    default:
      fatalError("unexpected string")
    }
  }
}

extension PackageDescription.Package {
  public static func fromTOML(item: TOMLItem, baseURL: String? = nil) -> PackageDescription.Package {
        ...
    var pkgConfig: String? = nil
    if case .String(let value)? = table.items["pkgConfig"] {
      pkgConfig = value
    }
    var providers: [PackageDescription.SystemPackageProvider]? = nil
    if case .Array(let array)? = table.items["providers"] {
      providers = []
      for item in array.items {
        providers?.append(PackageDescription.SystemPackageProvider.fromTOML(item))
      }
    }
        ...
  }
}

Now, lets create a class called PkgConfig that will wrap the pkg-config’s functionality. It’ll throw error if pkg-config or the system package is not installed.

final class PkgConfig {
  let cliName = "pkg-config"
  let name: String
  init(name: String) throws {
    guard (try? Utility.popen(["which", cliName])) != nil else {
      throw Error.PkgConfigNotInstalled
    }
    do {
      try system([cliName, "--exists", name])
    } catch {
      throw Error.SystemPackageNotInstalled(name)
    }
    self.name = name
  }
  
  var cFlags: [String] {
    return PkgConfig.runPopen([cliName, "--cflags", name]).chomp().characters.split(separator: " ").map{String($0)}
  }
  
  var libs: [String] {
    return PkgConfig.runPopen([cliName, "--libs", name]).chomp().characters.split(separator: " ").map{String($0)}
  }
  
  static private func runPopen(arguments: [String]) -> String {
    return (try? popen(arguments)) ?? ""
  }
}

extension PkgConfig {
  enum Error: ErrorProtocol {
    case PkgConfigNotInstalled
    case SystemPackageNotInstalled(String)
  }
}

We need to display the right provider help in case system module is not installed. Create an extension in Build module which would take array of the provider enums and try to figure out the provider for current platform.

extension SystemPackageProvider {
  var installText: String {
      switch self {
      case .Brew(let name):
        return "    brew install \(name)\n"
      case .Apt(let name):
        return "    apt-get install \(name)\n"
      }
  }
  static func providerForCurrentPlatform(providers: [SystemPackageProvider]) -> SystemPackageProvider? {
    guard let uname = try? popen(["uname"]).chomp().lowercased() else { return nil }
    switch uname {
    case "darwin":
      for provider in providers {
        if case .Brew = provider {
            return provider
          }
      }
    case "linux":
      if "/etc/debian_version".isFile {
        for provider in providers {
          if case .Apt = provider {
            return provider
          }
        }
      }
      break
    default:
      return nil
    }
    return nil
  }
}

Only thing left is asking PkgConfig what flags to add for each CModule which defined a pkgConfig name and append those flags in build params.

extension SwiftModule {
  var pkgConfigArgs: [String] {
    return recursiveDependencies.flatMap { module -> [String] in
      guard case let module as CModule = module, let pkgConfigName = module.pkgConfig else {
        return []
      }
      do {
        let pkgConfig = try PkgConfig(name: pkgConfigName)
        return pkgConfig.cFlags.map{["-Xcc", $0]}.flatten() + pkgConfig.libs
      } catch PkgConfig.Error.PkgConfigNotInstalled {
        print("warning: pkg-config is not installed.")
      } catch PkgConfig.Error.SystemPackageNotInstalled(let name) {
        print("System module \(name) not installed.")   
        if let providers = module.providers, provider = SystemPackageProvider.providerForCurrentPlatform(providers) {
          print("note: you may be able to install \(name) using your system-packager:\n")
          print(provider.installText)
        }            
      } catch {}
      return []
    }
  }
}

Done. Lets just try a “Hello World” Gtk program :

import CGtk
gtk_init(nil, nil)
let window = gtk_window_new(GTK_WINDOW_TOPLEVEL)
let button = gtk_button_new_with_label ("Hello World")
gtk_container_add(UnsafeMutablePointer(window), button)
gtk_widget_show(button)
gtk_widget_show(window)
gtk_main()

$ sb #sb is alias to swift-build

uh oh!

$ brew install gtk+3
🍺  /usr/local/Cellar/gtk+3/3.18.9: 1,334 files, 66.3M
$ sb
Compiling Swift Module 'SwiftGtk' (1 sources)
Linking .build/debug/SwiftGtk
$ .build/debug/SwiftGtk

Works!

Doing the same thing in Ubuntu:

$ sb #sb is alias to swift-build

$ sudo apt-get install gtk+3
$ sb
Compiling Swift Module 'SwiftGtk' (1 sources)
Linking .build/debug/SwiftGtk
$ .build/debug/SwiftGtk

PS: This was just for experimental purposes, the proposal is in draft state right now and might have some changes before its accepted or might not get accepted at all.